validation loss and training loss is not increasing. I try to imply l2 regularization in form of weight decay as 0.05 but I have removed and tried maybe that could be the reason. I have even removed the initialization of weights

    class CustomDataset(Dataset):

        def __init__(self, root_folder_path):

            self.root_folder_path = root_folder_path
            self.image_files = []
            self.labels = []

            # Collect image paths and corresponding labels

            folders = sorted([f for f in os.listdir(root_folder_path) if os.path.isdir(os.path.join(root_folder_path, f))])
            self.label_dict = {folder: i for i, folder in enumerate(folders)}

            for folder in folders:

                folder_path = os.path.join(root_folder_path, folder)
                image_files = sorted([f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f)) and f.endswith('.jpg')])
                self.image_files.extend([os.path.join(folder_path, img) for img in image_files])
                self.labels.extend([self.label_dict[folder]] * len(image_files))

            self.transform = transforms.Compose([
                transforms.Resize((900, 300)),
                transforms.Normalize(mean=[0.5], std=[0.5])


        def __len__(self):

            return len(self.image_files)

        def __getitem__(self, idx):

            image_path = self.image_files[idx]
            label = self.labels[idx]
            image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
            image = self.transform(image)
            #print("Image shape:", image.shape)  # Print the shape of the image
            one_hot_label = torch.zeros(len(self.label_dict))
            one_hot_label[label] = 1

            return image, one_hot_label

main script

from custom_dataset import CustomDataset

if __name__ == '__main__':
     # Instantiate your custom dataset and dataloaders
    root_folder_path = r'W:\MASTER_BAGCHI_SCHALDACH\THESIS\code and dataset\image_dataset_300_900_10_classes'
    dataset = CustomDataset(root_folder_path)

    print("Labels:", sorted(dataset.label_dict.keys()))
    print("Total number of labels:", len(dataset.label_dict))

    # Display some images from each folder
    n_images_to_display = 4
    n_folders = len(dataset.label_dict)
    fig, ax = plt.subplots(n_images_to_display, n_folders, figsize=(n_folders * 4, n_images_to_display * 4))

    for i, (folder, label) in enumerate(dataset.label_dict.items()):
        folder_images = [dataset[i][0] for i, lbl in enumerate(dataset.labels) if lbl == label]
        indices_to_display = random.sample(range(len(folder_images)), min(n_images_to_display, len(folder_images)))
        for j, ind in enumerate(indices_to_display):
            ax[j, i].imshow(folder_images[ind].squeeze(), cmap='gray')  # Squeeze to remove the channel dimension for grayscale images
            ax[j, i].axis('off')
        ax[0, i].set_title(folder, fontsize=30)

    fig.tight_layout(pad=0, w_pad=0, h_pad=0)


    from torch.utils.data import DataLoader, Subset
    from sklearn.model_selection import train_test_split

    TEST_SIZE = 0.2
    BATCH_SIZE = 32
    SEED = 42

    # Get the labels from the dataset
    labels = np.array([label for _, label in dataset])

    # generate indices: instead of the actual data we pass in integers instead
    train_indices, test_indices, _, _ = train_test_split(

    # generate subset based on indices
    train_split = Subset(dataset, train_indices)
    test_split = Subset(dataset, test_indices)
    print('Length of train_batch:',len(train_split))
    print('Length of test_batch:',len(test_split))

    # create batches
    train_loader = DataLoader(train_split, batch_size=BATCH_SIZE, num_workers=6,shuffle=True,pin_memory=True)
    test_loader = DataLoader(test_split, batch_size=BATCH_SIZE,num_workers=6,pin_memory=True)

    for batch in train_loader:
        images, labels = batch
        #print('Train batch size:', images.size())
        #print('Shape of labels array:',labels.size())

    for batch in test_loader:
        images, labels = batch
        #print('Test batch size:', images.size())
        #print('Shape of labels array:',labels.size())
    class ImageClassificationBase(nn.Module):
        def training_step(self, batch):
            images, labels = batch 
            out = self(images)                  # Generate predictions
            loss = F.cross_entropy(out, labels) # Calculate loss
            return loss
        def accuracy(self,outputs, labels):
            #_, preds = torch.max(outputs, dim=1)
            preds = torch.argmax(outputs, dim=1)
            #preds_one_hot = F.one_hot(preds, num_classes=labels.shape[1])  # Convert predictions to one-hot encoding
            #print("Shape of preds:", preds.shape)  # Check the shape of preds
            #correct=(preds_one_hot == labels).float().sum() # Count the number of correct predictions
            correct = (preds == torch.argmax(labels, dim=1)).float().sum()  # Count the number of correct predictions
            total = len(labels)  # Total number of samples
            acc = correct / total  # Calculate accuracy
            return acc
            #return torch.sum(preds_one_hot == labels).float().mean()          
        def validation_step(self, batch):
            images, labels = batch 
            out = self(images)                    # Generate predictions
            loss = F.cross_entropy(out, labels)   # Calculate loss
            acc = self.accuracy(out, labels)           # Calculate accuracy
            #batch_size = labels.shape[0]
            #acc = self.accuracy(out, labels, batch_size)
            return {'val_loss': loss.detach(), 'val_acc': acc}
        def validation_epoch_end(self, outputs):
            batch_losses = [x['val_loss'] for x in outputs]
            epoch_loss = torch.stack(batch_losses).mean()   # Combine losses
            batch_accs = [x['val_acc'] for x in outputs]
            epoch_acc = torch.stack(batch_accs).mean()      # Combine accuracies
            return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}
        def epoch_end(self, epoch, result):
            print("Epoch [{}], train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}".format(
                epoch, result['train_loss'], result['val_loss'], result['val_acc']))

    import torch.nn.init as init
    class ImageClassification(ImageClassificationBase):
        def __init__(self):
            self.network = nn.Sequential(
                #image size is [1,900,300] as [channel, height,width]
                nn.Conv2d(1, 32, kernel_size = 3, padding = 1),
                nn.AvgPool2d(kernel_size=2, stride=2),

                nn.Conv2d(32,32, kernel_size = 3,  padding = 1),
                nn.AvgPool2d(kernel_size=2, stride=2),
                nn.Conv2d(32, 64, kernel_size = 3, padding = 1),
                nn.AvgPool2d(kernel_size=2, stride=2),
                nn.Conv2d(64 ,64, kernel_size = 3, padding = 1),
                nn.AvgPool2d(kernel_size=2, stride=2),

                nn.Linear(64 * 56 * 18, 64),  # Assuming input size after convolutional layers is 64 * 56 * 18
                nn.Linear(64, 64),
                nn.Linear(64, 10)  # Output layer

        def forward(self, xb):
            return self.network(xb)

    def get_default_device():
        #Set Device to GPU or CPU
        if torch.cuda.is_available():
            return torch.device('cuda')
            return torch.device('cpu')

    def to_device(data, device):
        "Move data to the device"
        if isinstance(data,(list,tuple)):
            return [to_device(x,device) for x in data]
        return data.to(device,non_blocking = True)

    class DeviceDataLoader():
        #Wrap a dataloader to move data to a device
        def __init__(self, dl, device):
            self.dl = dl
            self.device = device
        def __iter__(self):
            #Yield a batch of data after moving it to device
            for b in self.dl:
                yield to_device(b,self.device)
        def __len__(self):
            #Number of batches
            return len(self.dl)

    device = get_default_device()

    model = ImageClassification()

    random_seed = 42

    train_loader = DeviceDataLoader(train_loader, device)
    test_loader = DeviceDataLoader(test_loader, device)

    to_device(model, device)

    def evaluate(model, test_loader):
        outputs = [model.validation_step(batch) for batch in test_loader]
        return model.validation_epoch_end(outputs)
    # Define the RMSprop optimizer
    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.001, alpha=0.99, eps=1e-08, momentum=0.9)
    from torch.optim.lr_scheduler import LambdaLR

    # Define the custom scheduler function
    def lr_schedule(epoch, lr):
        if epoch < 10:
            return lr
            return lr * torch.exp(torch.tensor(-0.1))

    # Create a LambdaLR scheduler using the custom function
    scheduler = LambdaLR(optimizer, lr_lambda=lambda epoch: lr_schedule(epoch, lr=0.001))

    def fit(epochs, model, train_loader, test_loader, optimizer):
        history = []
        for epoch in range(epochs):
            # Training Phase 
            train_losses = []

            correct_train = 0
            total_train = 0

            for batch in train_loader:
                #images,labels = batch
                #out = model(images)
                #loss = F.cross_entropy(out,labels)
                loss = model.training_step(batch)

                # Calculate training accuracy
                #preds = torch.argmax(out, dim=1)
                #correct_train += (preds == torch.argmax(labels, dim=1)).sum().item()
                #total_train += labels.size(0)


            # Validation phase
            result = evaluate(model, test_loader)
            result['train_loss'] = torch.stack(train_losses).mean().item()
            #result['train_loss'] = torch.tensor(train_losses).mean().item()
            #result['train_acc'] = correct_train / total_train
            model.epoch_end(epoch, result)
        return history


    #initial evaluation of the model
    # Initial evaluation of the model
    initial_result = evaluate(model, test_loader)
    accuracy_percentage = initial_result['val_acc'] * 100
    print('Initial Test Loss: {:.4f}, Initial Test Accuracy: {:.4f}%'.format(initial_result['val_loss'], accuracy_percentage))

    #set the no. of epochs, optimizer funtion and learning rate
    num_epochs = 10
    #fitting the model on training data and record the result after each epoch
    history = fit(num_epochs, model, train_loader, test_loader, optimizer)


Labels: ['120', '144', '168', '192', '216', '24', '240', '48', '72', '96']
Total number of labels: 10
Length of train_batch: 1835
Length of test_batch: 459
Initial Test Loss: 2.3067, Initial Test Accuracy: 9.5833%
Epoch [0], train_loss: 2.4796, val_loss: 2.3525, val_acc: 0.1229
Epoch [1], train_loss: 2.4644, val_loss: 2.3175, val_acc: 0.1188
Epoch [2], train_loss: 2.4614, val_loss: 2.3247, val_acc: 0.1083
Epoch [3], train_loss: 2.4695, val_loss: 2.3192, val_acc: 0.1167
Epoch [4], train_loss: 2.4771, val_loss: 2.3155, val_acc: 0.1292
Epoch [5], train_loss: 2.4994, val_loss: 2.3175, val_acc: 0.1292
Epoch [6], train_loss: 2.4528, val_loss: 2.3189, val_acc: 0.1125
Epoch [7], train_loss: 2.4887, val_loss: 2.3146, val_acc: 0.1331
Epoch [8], train_loss: 2.4908, val_loss: 2.3149, val_acc: 0.1208
Epoch [9], train_loss: 2.4809, val_loss: 2.3195, val_acc: 0.1208

Could someone please help why the model is not training?

  • $\begingroup$ Try starting with overfitting the model on a single batch of data, to make sure the model is able to actually learn patterns from the data. See also the tips mentioned by Andrej Karpathy in his blog post A Recipe for Training Neural Networks. $\endgroup$
    – Oxbowerce
    Mar 9 at 16:00
  • $\begingroup$ thankyou @Oxbowerce for the reply. So how to model it for single batch of data and my model is already running on tensorflow. I want to try torch because the gpu memory allocation is better in torch compared to tensorflow $\endgroup$ Mar 9 at 16:05
  • $\begingroup$ you can check if you can overfit by training on a much smaller dataset. For example, subset your dataset to only 10 images. In this dataset, you should be able to get low training loss, if not you have a problem with the training script. $\endgroup$ Mar 11 at 6:30
  • $\begingroup$ @saiRegrefree - I have tried on a small data set and it is giving the same loss so there is some problem in the loss but I couldn't figure it out $\endgroup$ Mar 11 at 11:50


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.