0
$\begingroup$

I have a multi-modal model. I want to train it using the Pytorch Framework. I have a balanced dataset. I have approximately 150 samples for each client. (I had preprocessed my text data.) when I train my model it doesn't learn anything. This is my custom multi-modal model class:

class MultiModalModel(nn.Module):
def __init__(self, num_classes):
    super(MultiModalModel, self).__init__()

    self.resnet = resnet50(pretrained=True)
    self.image_branch = nn.Sequential(
        *list(self.resnet.children())[:-2],
    )

    num_layers_to_unfreeze = 30
    start_layer_index = len(list(self.image_branch.children())) - num_layers_to_unfreeze

    for layer_index, param in enumerate(self.image_branch.parameters()):
        if layer_index >= start_layer_index:
          param.requires_grad = True
        else:
          param.requires_grad = False

    # Text branch
    # self.bert_model = AutoModel.from_pretrained("vinai/bertweet-base")
    self.bert_model = BertModel.from_pretrained("bert-base-uncased")

    total_layers = len(list(self.bert_model.children()))

    # Specify the number of layers you want to unfreeze from the end
    num_layers_to_unfreeze = 20

    # Calculate the starting index of the layers to unfreeze
    start_layer_index = total_layers - num_layers_to_unfreeze

    # Iterate over the parameters and unfreeze the last 10 layers
    for layer_index, param in enumerate(self.bert_model.parameters()):
        if layer_index >= start_layer_index:

          param.requires_grad = True
        else:
          param.requires_grad = False

    # Fusion layer
    self.fusion_layer = nn.Linear(2048 + self.bert_model.config.hidden_size, 10)  # Adjust input size based on your needs

    self.hidden1 = nn.Linear(10, 10)
    self.hidden2 = nn.Linear(10, 10)
    self.hidden3 = nn.Linear(10, 10)

    self.dropout = nn.Dropout(p=0.7)
    # Output layer
    self.output_layer = nn.Linear(10, num_classes)
    
def forward(self, image_input, input_ids, attention_mask, token_type_ids=0):
    # Image branch (ResNet)
    image_features = self.image_branch(image_input)
    image_features = F.adaptive_avg_pool2d(image_features, (1, 1))
    image_features = image_features.view(image_features.size(0), -1)


    # Text branch (BERT)
    # _, pooled_output = self.bert_model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids,return_dict=False)
    _, pooled_output = self.bert_model(input_ids=input_ids, attention_mask=attention_mask, return_dict=False)

    # Concatenate features from both branches
    fused_features = torch.cat((image_features, pooled_output), dim=1)
    # Fusion layer
    fused_features = F.relu(self.fusion_layer(fused_features))
    fused_features = self.dropout(fused_features)

    #add hidden layer
    hidden1 = F.relu(self.hidden1(fused_features))
    hidden1 = self.dropout(hidden1)

    hidden2 = F.relu(self.hidden2(hidden1))
    hidden2 = self.dropout(hidden2)

    hidden3 = F.relu(self.hidden3(hidden2))
    hidden3 = self.dropout(hidden3)
    
    output = self.output_layer(hidden3)
    return output

and this is my client class that train individual clients:

class Client():
def __init__(self, client_id, train_data, val_data, model, weights, epochs, optimizer, loss_func, scheduler=None, batch_size=32):
    self.client_name = "client_" + str(client_id)
    self.train_sampler = self.sampler(train_data)
    self.val_sampler = self.sampler(val_data)
    self.train_data = DataLoader(CustomDataset(pd.DataFrame.from_dict(train_data), transforms), shuffle=False, batch_size=batch_size, sampler=self.train_sampler)
    self.val_data = DataLoader(CustomDataset(pd.DataFrame.from_dict(val_data), transforms), shuffle=False, batch_size=batch_size, sampler=self.val_sampler)

    self.model = model
    self.epochs = epochs
    self.optimizer = optimizer
    self.scheduler = scheduler
    self.loss_func = loss_func
    self.model.load_state_dict(weights)
def sampler(self , data):
labels = data['label']
class_weights = 1.0 / torch.bincount(torch.tensor(data['label']))
# Create a weight for each sample
weights = class_weights[labels]
return WeightedRandomSampler(weights, len(weights))

def validation(self, valid_data):
val_loss = []
val_f1_score = []

all_labels = []
all_predictions = []

total_val_loss = 0
for idx, batch in enumerate(valid_data):
  img = batch['image'].to(device)

  input_ids = batch['text']['input_ids'].to(device)
  attention_mask = batch['text']['attention_mask'].to(device)
  token_type_ids = batch['text']['token_type_ids'].to(device)
  lbls = batch['label'].to(device)

  valid_output = self.model(img, input_ids, attention_mask, token_type_ids)
  y_pred = torch.argmax(valid_output,dim=1).cpu()

  all_labels.append(lbls)
  all_predictions.append(y_pred)

  vloss = self.loss_func(valid_output, lbls)

  total_val_loss += vloss

all_labels = torch.cat(all_labels).cpu().numpy()
all_predictions = torch.cat(all_predictions).cpu().numpy()

f1_score_value = f1_score(all_labels, all_predictions, average='weighted')
val_precision_value = precision_score(all_labels, all_predictions, average='weighted')
val_recall_value = recall_score(all_labels, all_predictions, average='weighted')
val_balanced_accuracy_score_value = accuracy_score(all_labels, all_predictions)

avg_val_loss = total_val_loss / len(valid_data)
avg_f1_score = torch.mean(torch.tensor(f1_score_value))
avg_precision = torch.mean(torch.tensor(val_precision_value))
avg_recall = torch.mean(torch.tensor(val_recall_value))
avg_acc = torch.mean(torch.tensor(val_balanced_accuracy_score_value))


return avg_val_loss, avg_f1_score, avg_precision, avg_recall, avg_acc

def train(self):

if torch.cuda.is_available():
  self.model = self.model.cuda()
  self.loss_func = self.loss_func.cuda()

train_loss = []
train_acc = []
train_f1 = []

val_loss = []
val_acc = []
val_f1 = []
val_f1_score = []
val_precision = []
val_recall = []
for epoch_i in range(0, self.epochs):
# ========================================
#               Training
# ========================================
  print(f'======== Epoch {epoch_i + 1} / {self.epochs} ========')
  print(f"Client ID : {self.client_name}")
  print('Training...')

  all_labels = []
  all_predictions = []

  epoch_train_loss = []
  epoch_train_acc = []
  train_f1_score = []
  train_precision = []
  train_recall = []

  total_train_loss = 0

  self.model.train(True)

  for step, batch in tqdm(enumerate(self.train_data)):

      image = batch['image'].to(device)
      input_ids = batch['text']['input_ids'].to(device)
      attention_mask = batch['text']['attention_mask'].to(device)
      token_type_ids = batch['text']['token_type_ids'].to(device)
      lbl = batch['label'].to(device)

      self.model.zero_grad()

      # result = self.model(image, input_ids, attention_mask, token_type_ids)
      result = self.model(image, input_ids, attention_mask)

      y_pred = torch.argmax(result,dim=1).cpu()
      all_labels.append(lbl)
      all_predictions.append(y_pred)


      loss = self.loss_func(result, lbl)
      total_train_loss += loss.item()


      # Perform a backward pass to calculate the gradients.
      loss.backward()
      # Clip the norm of the gradients to 1.0.
      # This is to help prevent the "exploding gradients" problem.
      torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
      self.optimizer.step()

  if self.scheduler:
    self.scheduler.step()
  epoch_train_loss.append(total_train_loss / len(self.train_data))

  all_labels = torch.cat(all_labels).cpu().numpy()
  all_predictions = torch.cat(all_predictions).cpu().numpy()

  train_f1_score_value = f1_score(all_labels, all_predictions, average='weighted')
  train_precision_value = precision_score(all_labels, all_predictions, average='weighted')
  train_recall_value = recall_score(all_labels, all_predictions, average='weighted')
  train_balanced_accuracy_score_value = accuracy_score(all_labels, all_predictions) #for imbalanced dataset

  train_loss_avg = torch.mean(torch.tensor(epoch_train_loss))
  train_f1_avg = torch.mean(torch.tensor(train_f1_score_value))
  train_acc_avg = torch.mean(torch.tensor(train_balanced_accuracy_score_value))
  train_precision_avg = torch.mean(torch.tensor(train_precision_value))
  train_recall_avg = torch.mean(torch.tensor(train_recall_value))

  train_loss.append(train_loss_avg)
  train_f1.append(train_f1_avg)
  train_acc.append(train_acc_avg)
  train_precision.append(train_precision_avg)
  train_recall.append(train_recall_avg)

  print(f"Average training f1_score: {train_f1_avg :.3f}")
  print(f"Average training balanced accuracy: {train_acc_avg :.3f}")
  print(f"Average training precision: {train_precision_avg :.3f}")
  print(f"Average training recall: {train_recall_avg :.3f}")
  print(f"Average training loss: {train_loss_avg :.3f}")

  with torch.no_grad():
    print("________validation metrics__________")
    val_avg_loss, val_avg_f1_score, val_avg_precision, val_avg_recall, val_avg_acc = self.validation(self.val_data)
    val_loss.append(val_avg_loss)
    val_acc.append(val_avg_acc)
    val_f1_score.append(val_avg_f1_score)
    val_precision.append(val_avg_precision)
    val_recall.append(val_avg_recall)

    print(f"Average validation f1_Score: {val_avg_f1_score :.3f}")
    print(f"Average validation balanced accuracy: {val_avg_acc :.3f}")
    print(f"Average validation precision: {val_avg_precision :.3f}")
    print(f"Average validation recall: {val_avg_recall :.3f}")
    print(f"Average validation loss: {val_avg_loss :.3f}")

#set new weights
weights = self.model.state_dict()
return {'client_name': self.client_name,
        'weights': weights,
        'train_loss': train_loss,
        'train_f1_scores': train_f1,
        'train_balanced_accuracy': train_acc,
        'train_precision': train_precision,
        'train_recall': train_recall,

        'val_loss': val_loss,
        'val_f1_score': val_f1_score,
        'val_balanced_accuracy': val_acc,
        'val_precision': val_precision,
        'val_recall': val_recall
        }

and this is my optimizer and loss-function setting :

from torch.optim.lr_scheduler import ExponentialLR

optimizer = torch.optim.Adadelta(model.parameters(),
              lr = 1e-3, # args.learning_rate - default is 5e-5, our notebook had 2e-5
              eps = 1e-06, # args.adam_epsilon  - default is 1e-8.
              weight_decay=1e-5
            )

scheduler = ExponentialLR(optimizer, gamma=0.9)

loss_fn = nn.CrossEntropyLoss()
rounds = 3
epochs = 10
batch_size = 4
server = Server(encoded_train, encoded_val, encoded_test, model, rounds, epochs, optimizer, scheduler, nn.CrossEntropyLoss(), clients_number=num_clients, batch_size=batch_size)
train_res = server.train()

and this is my loss values and accuracy per epoch: average_loss_values_per_epoch average_accuracy_per_epoch please help me what should I do?

$\endgroup$
4
  • $\begingroup$ A usual approach to debug models is trying to overfit them to a single batch to see if they work. $\endgroup$
    – noe
    Jan 23 at 23:59
  • $\begingroup$ @noe I modify my dataset to binary dataset and just using text data for classification and I get 50% accuracy. is it ok?(my data is tweet) $\endgroup$ Jan 24 at 11:18
  • $\begingroup$ Having 50% accuracy for a binary classifier on a balanced dataset is equivalent to random selection, so no, not Ok. $\endgroup$
    – noe
    Jan 24 at 11:24
  • $\begingroup$ @noe so what should I do?I tested every thing, thats ok. $\endgroup$ Jan 24 at 22:11

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.