Why my simple resnet model overfit?

Question

I work on data classification. My train results are good 90%+ accuracy, but the test accuracy/loss is inconsistent. I don't succeed to get rid of the overfitting. The images are grouped, so to classify a group in the testing phase, I take the dominant label from the group.

Epoch 1/13 | train loss 40.1072 | train acc 58.04% | test loss 1.4724 | test acc 21.43%
Epoch 2/13 | train loss 33.5286 | train acc 60.79% | test loss 1.2567 | test acc 57.14%
Epoch 3/13 | train loss 29.6434 | train acc 64.11% | test loss 1.6774 | test acc 28.57%
Epoch 4/13 | train loss 24.6312 | train acc 71.13% | test loss 1.3042 | test acc 57.14%
Epoch 5/13 | train loss 20.2970 | train acc 75.77% | test loss 1.6517 | test acc 50.00%
Epoch 6/13 | train loss 18.2522 | train acc 78.53% | test loss 1.5980 | test acc 50.00%
Epoch 7/13 | train loss 15.7128 | train acc 81.92% | test loss 1.7192 | test acc 42.86%
Epoch 8/13 | train loss 13.6410 | train acc 84.00% | test loss 1.8424 | test acc 28.57%
Epoch 9/13 | train loss 12.2350 | train acc 85.55% | test loss 1.5387 | test acc 57.14%
Epoch 10/13 | train loss 11.6166 | train acc 86.53% | test loss 2.2462 | test acc 28.57%
Epoch 11/13 | train loss 9.1678 | train acc 89.55% | test loss 3.2351 | test acc 14.29%
Epoch 12/13 | train loss 8.7523 | train acc 89.70% | test loss 1.6232 | test acc 28.57%
Epoch 13/13 | train loss 7.7750 | train acc 90.83% | test loss 2.1384 | test acc 21.43%

These results are with a resize to 128*128, and dropout layer with p=0.5.

The model is build like this:

model = models.resnet18(pretrained=True)
fc_layer_list = []
fc_layer_list.append(nn.Dropout(p=params.dropout_layer_prob))
fc_layer_list.append(nn.Linear(model.fc.in_features, params.num_of_classes))
model.fc = nn.Sequential(*fc_layer_list)
model = model.to(params.device)

for param in model.parameters():
    param.requires_grad = False
for param in model.fc.parameters():
    param.requires_grad = True

optimizer_params = model.fc.parameters()
optimizer = optim.Adam(optimizer_params, lr=params.lr)

train code:

def train_model():
    epoch_range = range(params.epochs)
    for epoch in epoch_range:
        model.train()
        start_time = time.time()
        y_true = torch.empty(len(train_loader.dataset), dtype=int)
        y_pred = torch.empty(len(train_loader.dataset), dtype=int)
        cur_group_idx = 0
        epoch_loss = 0.0
        for batch_idx, (data, targets) in enumerate(train_loader):
            optimizer.zero_grad()
            batch_accumulative_loss = 0.0
            for image, target in zip(data, targets):
                y_true[cur_group_idx] = target.item()
                device_image = image.to(params.device)
                device_target = target.to(params.device)
                output = model(device_image.unsqueeze(0)).squeeze(0)
                y_pred[cur_group_idx] = output.argmax().item()
                minibatch_loss = F.cross_entropy(output, device_target)
                batch_accumulative_loss += minibatch_loss
                del device_image, device_target, output
                torch.cuda.empty_cache()
                cur_group_idx += 1
            batch_accumulative_loss.backward()
            optimizer.step()
            epoch_loss += batch_accumulative_loss.item()
            del batch_accumulative_loss
            torch.cuda.empty_cache()
        epoch_loss /= len(train_loader)
        if scheduler is not None:
            scheduler.step()
        test_acc, test_loss = test_model()
        acc = accuracy_score(y_true, y_pred)

Test:

def test_model():
    model.eval()
    with torch.no_grad():
        accumulative_loss = 0.0
        y_true = torch.empty(len(test_loader.dataset), dtype=int)
        y_pred = torch.empty(len(test_loader.dataset), dtype=int)
        cur_group_idx = 0
        total_amount_of_images = 0
        for groups_images_batch, labels_batch in test_loader:
            for group_images, label in zip(groups_images_batch, labels_batch):
                device_label = label.to(params.device)
                y_true[cur_group_idx] = label.item()
                preds_per_image = torch.empty(len(group_images), dtype=int)
                for image_idx, image in enumerate(group_images):
                    total_amount_of_images += 1
                    device_image = image.to(params.device)
                    out = model(device_image.unsqueeze(0)).squeeze(0)
                    preds_per_image[image_idx] = out.argmax().item()
                    loss = F.cross_entropy(out, device_label)
                    accumulative_loss += loss.item()
                    if params.device == 'cuda':
                        del out, device_image, loss
                        torch.cuda.empty_cache()
                y_pred[cur_group_idx] = preds_per_image.mode().values.item()
                cur_group_idx += 1
                del device_label
                torch.cuda.empty_cache()
        acc = accuracy_score(y_true, y_pred)
        avg_loss = accumulative_loss / total_amount_of_images
    return acc, avg_loss

Data classes:

label 0: train: 32, test: 8
label 1: train: 7, test: 2
label 2: train: 5, test: 2
label 3: train: 7, test: 2
label 4: train: 2, test: 0

Answer 1

It looks like your dataset is tiny, so I wouldn't be surprised if your model picks the majority class most of the time.

Here are a few things I'd do:

• Increase amount of data if possible
• Analyze prediction scores for your test data (i.e., is your model "hesitating," or is it attributing the majority of the score to a particular class?)
• Try different train-test splits to see if this is just an anomaly with the training data you have