Aspect-Based Sentiment Analysis with Bert and Pytorch

Question

I have a dataset of online reviews (X) with their corresponding topics (topic1 to topic5) and each topic can have 5 values (fined-grained sentiment score from 1 to 5). So, I have one X and 5 Y columns. I was wondering how can I use Bert and Pytorch to train a model which gets textual data and make an output like ([2,3,1,5,4] meaning topic1: 2 topic2: 3 and so on). Currently, my solution is like this but my metrics are not good. I really appreciate it if you help me with dealing with the imbalance situation since for every topic, scores 1 and 2 are really small.

class SentimentClassifier(nn.Module):
  def __init__(self, n_classes):
    super(SentimentClassifier, self).__init__()
    self.bert = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)
    self.drop = nn.Dropout(p=0.1)
    self.out = nn.Linear(self.bert.config.hidden_size, n_classes)
  def forward(self, input_ids, attention_mask):
    _, pooled_output = self.bert(
      input_ids=input_ids,
      attention_mask=attention_mask, return_dict=False
    )
    output = self.drop(pooled_output)
    output_1 = self.out(output)
    output_2 = self.out(output)
    output_3 = self.out(output)
    output_4 = self.out(output)
    output_5 = self.out(output)
    return output_1 ,output_2 ,output_3 ,output_4 ,output_5 

torch.cuda.manual_seed(3447)
def train_epoch(
  model,
  data_loader,
  loss_fn,
  optimizer,
  device,
  scheduler
):
  model = model.train()
  losses = []
  acc = []
  f1 = []
  for d in data_loader:
    input_ids = d["input_ids"].to(device)
    attention_mask = d["attention_mask"].to(device)
    target_1 = d["targets"][:,0].to(device)
    target_2 = d["targets"][:,1].to(device)
    target_3 = d["targets"][:,2].to(device)
    target_4 = d["targets"][:,3].to(device)
    target_5 = d["targets"][:,4].to(device)
    output_1 ,output_2 ,output_3 ,output_4 ,output_5 = model(
      input_ids=input_ids,
      attention_mask=attention_mask
    )
    preds_1 = torch.argmax(output_1 , dim=1)
    preds_2 = torch.argmax(output_2 , dim=1)
    preds_3 = torch.argmax(output_3 , dim=1)
    preds_4 = torch.argmax(output_4 , dim=1)
    preds_5 = torch.argmax(output_5 , dim=1)

    loss_1 = loss_fn(output_1 , target_1 -1)
    loss_2 = loss_fn(output_2 , target_2 -1)
    loss_3 = loss_fn(output_3 , target_3 -1)
    loss_4 = loss_fn(output_4 , target_4 -1)
    loss_5 = loss_fn(output_5 , target_5 -1)
    loss = loss_1 + loss_2 + loss_3 + loss_4 + loss_5 

    acc_1 = accuracy_score(preds_1 ,target_1 -1).item()
    f1_1 = f1_score(preds_1 ,target_1 -1).item()
    ....(for other Ys)


    
    acc_total = (acc_1 + acc_2 + acc_3 + acc_4 + acc_5) / 5
    f1_total = (f1_1 + f1_2 + f1_3 + f1_4 + f1_5) / 5

    losses.append(loss.item())
    acc.append(acc_total)
    f1.append(f1_total)

    loss.backward()

    nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
    optimizer.step()
    scheduler.step()
    optimizer.zero_grad()
    gc.collect()
    torch.cuda.empty_cache()
  return np.mean(losses), np.mean(acc), np.mean(f1) 

Do you think my approach is OK? I see the problem as a multi-output classification (5 multi-class problems), so I used a 5-head output for the deep learning architecture. I really appreciate any help, comment, and resource improving the model. Thank you so much

Answer 1

Add a dense layer on top of the pooled output and write a softmax with 5 units in it.

Also, Can you print your model summary and share it please!