Is it good if during training the model my test accuracy much higher then train accuracy. How can i prevent this?

Question

my training code:

import torch
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from torchvision import datasets, transforms


from CNN import CNNmodel

SEED = 5
device = "cuda" if torch.cuda.is_available() else "cpu"
BATCH_SIZE = 16

torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)

train_transform = transforms.Compose([
    transforms.TrivialAugmentWide(num_magnitude_bins=8),
    transforms.ToTensor()
])

test_transform = transforms.Compose([
    transforms.ToTensor()
])

train_data = datasets.MNIST(
    root="data",
    train=True,
    download=True,
    transform=train_transform
)

test_data = datasets.MNIST(
    root="data",
    train=False,
    download=True,
    transform=test_transform
)

train_dataloader = DataLoader(
    train_data,
    batch_size=BATCH_SIZE,
    shuffle=True
)

test_dataloader = DataLoader(
    test_data,
    batch_size=BATCH_SIZE,
    shuffle=False
)

channel_num = train_data[0][0].shape[0]
model = CNNmodel(in_shape=channel_num, hidden_shape=16, out_shape=len(train_data.classes)).to(device)
optimizer = torch.optim.SGD(params=model.parameters(), lr=0.01)
loss_fn = torch.nn.CrossEntropyLoss()
epochs = 20
writer = SummaryWriter(log_dir="runs\\CNN_MNIST")

for epoch in range(epochs):
    train_loss = 0
    train_acc = 0

    model.train()
    for batch, (X, y) in enumerate(train_dataloader):
        X, y = X.to(device), y.to(device)
        X = torch.reshape(X, (BATCH_SIZE, channel_num, 28, 28))

        y_pred = model(X)

        loss = loss_fn(y_pred, y)
        train_loss += loss.item()
        
        optimizer.zero_grad()

        loss.backward()

        optimizer.step()

        y_pred_class = torch.argmax(torch.softmax(y_pred, dim=1), dim=1)
        train_acc += (y_pred_class == y).sum().item()/len(y_pred)
    
    train_loss /= len(train_dataloader)
    train_acc /= len(train_dataloader)
    
    test_loss = 0
    test_acc = 0

    model.eval()
    with torch.inference_mode():
        for batch, (X, y) in enumerate(test_dataloader):
            X, y = X.to(device), y.to(device)

            y_pred = model(X)

            loss = loss_fn(y_pred, y)
            test_loss += loss.item()

            y_pred_class = torch.argmax(torch.softmax(y_pred, dim=1), dim=1)
            test_acc += (y_pred_class == y).sum().item()/len(y_pred)
    
    test_loss /= len(test_dataloader)
    test_acc /= len(test_dataloader)

   
    writer.add_scalars(
        main_tag="Loss",
        tag_scalar_dict={"train_loss": train_loss,
                        "test_loss": test_loss   },
        global_step=epoch
    )

    writer.add_scalars(
        main_tag="Accuracy", 
        tag_scalar_dict={"train_acc": train_acc,
                        "test_acc": test_acc   }, 
        global_step=epoch
    )

    writer.close()
    torch.cuda.empty_cache()
    print(f"epoch={epoch}, train loss={train_loss}, train acc={train_acc}, test loss={test_loss}, test acc={test_acc}\n")

torch.save(model.state_dict(), f="CNN.pth")

my results:

epoch=0, train loss=1.2992823556999364, train acc=0.5814166666666667, test loss=0.1535775218948722, test acc=0.9617

epoch=1, train loss=0.7351227536817392, train acc=0.7735333333333333, test loss=0.0957084314838983, test acc=0.9711

epoch=2, train loss=0.6108905077829957, train acc=0.8069666666666667, test loss=0.10527049974631518, test acc=0.968

epoch=3, train loss=0.5531635082634787, train acc=0.8209333333333333, test loss=0.09478655792670325, test acc=0.9719

epoch=4, train loss=0.5146081379964947, train acc=0.8315666666666667, test loss=0.10086005784235895, test acc=0.9717

epoch=5, train loss=0.48089857985948525, train acc=0.8415166666666667, test loss=0.07805026951334439, test acc=0.9755

epoch=6, train loss=0.46410337663746126, train acc=0.8458, test loss=0.06370123700092081, test acc=0.979

epoch=7, train loss=0.45169676643597584, train acc=0.8508333333333333, test loss=0.06549387282291427, test acc=0.9784

epoch=8, train loss=0.4308121643635134, train acc=0.8575, test loss=0.07395816469893325, test acc=0.9764

epoch=9, train loss=0.42585810295939447, train acc=0.8576166666666667, test loss=0.060803520213114096, test acc=0.9809

epoch=10, train loss=0.412179026115189, train acc=0.8625, test loss=0.05902050706697628, test acc=0.9811

epoch=11, train loss=0.4062708326317991, train acc=0.8628666666666667, test loss=0.05916510981819592, test acc=0.982

epoch=12, train loss=0.3950844133876264, train acc=0.8676666666666667, test loss=0.051657470285263844, test acc=0.9839

epoch=13, train loss=0.3960405339717865, train acc=0.8668666666666667, test loss=0.05090424774668645, test acc=0.9838

epoch=14, train loss=0.3826637831449664, train acc=0.8697333333333334, test loss=0.049632979356194845, test acc=0.9839

epoch=15, train loss=0.38186972920044016, train acc=0.87205, test loss=0.05163152083947789, test acc=0.9828

epoch=16, train loss=0.37976737998841953, train acc=0.8736166666666667, test loss=0.054158556177618444, test acc=0.9823

epoch=17, train loss=0.3711047379902874, train acc=0.8751333333333333, test loss=0.055461415114835835, test acc=0.9816

epoch=18, train loss=0.369529847216544, train acc=0.87475, test loss=0.046305917761620366, test acc=0.9861

epoch=19, train loss=0.3628049560392275, train acc=0.8773833333333333, test loss=0.05091290192245506, test acc=0.9846

Answer 1

Such a large gap suggests a problem with your test data. Maybe there is a data leak (i.e. part of your test data is leaked directly or indirectly into the training data).