Convolution neural network loss increasing instead of decreasing

Question

I am working on a binary image classification task in which I have greyscale images of size (1, 224, 224) (all normalized between 0 and 1) and a set of labels (0 or 1). I have around 2.6k images with labels, but while training the loss is increasing instead of decreasing.

This is my model:

import torch.nn as nn
import torch.nn.functional as F

class Classifier(nn.Module):
    def __init__(self):
        super().__init__()

        # Convolutional layers (1, 224, 224)
        self.conv1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=(3, 3), stride=1, padding=1)
        self.conv2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(3, 3), stride=1, padding=1)
        self.conv3 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 3), padding=1)

        # Fully connected layers
        self.fc1 = nn.Linear(in_features=64 * 28 * 28, out_features=512)
        self.fc2 = nn.Linear(in_features=512, out_features=256)
        self.fc3 = nn.Linear(in_features=256, out_features=32)
        self.fc4 = nn.Linear(in_features=32, out_features=1)

    def forward(self, X):
        X = F.relu(self.conv1(X))
        X = F.max_pool2d(X, 2)

        X = F.relu(self.conv2(X))
        X = F.max_pool2d(X, 2)

        X = F.relu(self.conv3(X))
        X = F.max_pool2d(X, 2)

        X = X.view(X.shape[0], -1)
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = F.relu(self.fc3(X))
        X = self.fc4(X)

        return X

and this is the training loop:

hope = Classifier()
loss_fn = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(hope.parameters(), lr=0.001)
epochs=100
hope = hope.to(device)
loss_fn = loss_fn.to(device)

for epoch in range(epochs):

    i=0
    loss_c=[]

    hope.train()

    for image,label in iter(ctrain_loader):
        image  = image.view((-1,1,224,224))
        pred = hope(image)
        label = label.view((-1,1))
        label = label.float()
        pred = pred.float()
        loss = loss_fn(pred,label)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
       
    hope.eval()
    
    for image,label in iter(cval_loader):
        image = image.view((-1,1,224,224))
        pred=hope(image)
        print(pred)
        label = label.view((-1,1))
        label = label.float()
        pred = pred.float()
        #pred = torch.sigmoid(pred)
        loss=loss_fn(pred,label)
        loss_c.append(loss.cpu().detach().numpy())

    print(f"loss is : {np.array(loss_c).mean()}")
         

I am using a batch size of 8 for training, i.e. labels supplied have dimensions (8,1) and images are of size (8,1,224,224).

I have tried switching to BCE but no help. Also, one thing I have observed is that for all the images in a batch the prediction values are always the same (might have something to do with initializing using random weights, but I haven't seen others doing it in their code so not sure).