1
$\begingroup$

I trained NLP models. This is a subset (200 instances) of my data set of 10,000 instances:This the link of the dataset on pastebin

I compare an LSTM model with a glove model and a BERT model. I expected a good performance with BERT. I can't get past 20% accuracy with BERT at all. I wonder what I'm missing in its implementation.

!pip list
#tensorflow 2.12.0
!python --version
#python 3.10.11

import json 
import tensorflow as tf
import numpy as np
from hyperopt import Trials, STATUS_OK, tpe
from sklearn.model_selection import train_test_split
from keras.layers import Input
from sklearn.metrics import accuracy_score
import pandas as pd

# Reading of file
f = open ('sampled_data.json', "r")
data = json.loads(f.read())

Data preprocessing

X=[x["title"].lower() for x in data]
y=[x["categories"][0].lower() for x in data]
X_train,X_test,y_train,y_test= train_test_split(X,y, test_size=0.2, 
                                                random_state=42)

target preprocessing. To consider the category unknown if not seen in test set

cat_to_id={'<UNK>':'0'}
for cat in y_train:
  if cat not in cat_to_id:
    cat_to_id[cat]=len(cat_to_id)

#MAPPING WITH RESPECT TO THE TRAINING SET
id_to_cat={v:k for k,v in cat_to_id.items()}
def preprocess_Y(Y,cat_to_id):
  res=[]
  for ex in Y:
    if ex not in cat_to_id.keys():
      res.append(cat_to_id['<UNK>'])
    else: 
      res.append(cat_to_id[ex])
  return np.array(res)

y_train_id=preprocess_Y(y_train,cat_to_id)
y_test_id=preprocess_Y(y_test,cat_to_id)
y_test_id=y_test_id.astype(float)

# Tokenization of of features
tokenizer=tf.keras.preprocessing.text.Tokenizer(num_words=10000)
tokenizer.fit_on_texts(X_train)

# TEXT TO SEQUENCE
X_train_seq=tokenizer.texts_to_sequences(X_train)
X_test_seq=tokenizer.texts_to_sequences(X_test)

#PADDING pad_sequences function transform in array
max_len=max([len(length) for length in X_train_seq])
X_train_pad= tf.keras.preprocessing.sequence.pad_sequences(X_train_seq,maxlen=max_len, truncating='post')
X_test_pad= tf.keras.preprocessing.sequence.pad_sequences(X_test_seq,maxlen=max_len, truncating='post')


####### RECCURRENT NEURAL NETWORK###############

vocab_size=len(tokenizer.word_index)
Embed_dim=300
dropout=0.2
dense_size=128
num_cat=len(cat_to_id)
batch_size=16
epochs=15

### CREER LE MODELE
model_rnn=tf.keras.models.Sequential()

# Add an embedding layer
model_rnn.add(tf.keras.layers.Embedding(input_dim=vocab_size, 
                                        output_dim=Embed_dim, 
                                        input_length=max_len))

# Add an LSTM layer
model_rnn.add(tf.keras.layers.LSTM(units=128))
model_rnn.add(tf.keras.layers.Dropout(0.4))

# Dense + activation
model_rnn.add(tf.keras.layers.Dense(units=dense_size,activation='relu'))
#Classifieur + activation
model_rnn.add(tf.keras.layers.Dense(units=num_cat,activation='softmax'))
print(model_rnn.summary())

model_rnn.compile(loss= 'sparse_categorical_crossentropy',
                  optimizer='adam',
                  metrics='accuracy')

model_rnn.fit(X_train_pad,y_train_id, batch_size=batch_size, epochs=epochs)

model_rnn.evaluate(X_test_pad, y_test_id)


Epoch 1/15
9/9 [==============================] - 5s 166ms/step - loss: 3.6699 - accuracy: 0.1643
Epoch 2/15
9/9 [==============================] - 1s 128ms/step - loss: 3.3861 - accuracy: 0.2286
Epoch 3/15
9/9 [==============================] - 1s 157ms/step - loss: 3.1313 - accuracy: 0.2357
Epoch 4/15
9/9 [==============================] - 1s 88ms/step - loss: 3.0774 - accuracy: 0.2286
Epoch 5/15
9/9 [==============================] - 1s 127ms/step - loss: 3.0358 - accuracy: 0.2286
Epoch 6/15
9/9 [==============================] - 0s 27ms/step - loss: 2.9461 - accuracy: 0.2286
Epoch 7/15
9/9 [==============================] - 0s 27ms/step - loss: 2.7970 - accuracy: 0.2357
Epoch 8/15
9/9 [==============================] - 1s 75ms/step - loss: 2.5048 - accuracy: 0.2429
Epoch 9/15
9/9 [==============================] - 1s 86ms/step - loss: 2.2543 - accuracy: 0.3357
Epoch 10/15
9/9 [==============================] - 1s 47ms/step - loss: 1.9985 - accuracy: 0.4357
Epoch 11/15
9/9 [==============================] - 0s 39ms/step - loss: 1.7728 - accuracy: 0.4929
Epoch 12/15
9/9 [==============================] - 1s 41ms/step - loss: 1.5552 - accuracy: 0.5929
Epoch 13/15
9/9 [==============================] - 0s 11ms/step - loss: 1.3320 - accuracy: 0.5929
Epoch 14/15
9/9 [==============================] - 0s 11ms/step - loss: 1.1506 - accuracy: 0.6786
Epoch 15/15
9/9 [==============================] - 0s 42ms/step - loss: 0.9498 - accuracy: 0.7714
2/2 [==============================] - 1s 13ms/step - loss: 6.6335 - accuracy: 0.2000



############# MODEL WITH GLOVE###################
embeddings_index = {}
f = open('glove.6B.300d.txt', encoding='utf-8')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

# Create embedding matrix
word_index=tokenizer.word_index
num_words = len(word_index) + 1
embedding_dim = 300
embedding_matrix = np.zeros((num_words, embedding_dim))
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector


    num_words=len(tokenizer.word_index)+1
embedding_dim=300
max_len=max([len(length) for length in X_train_seq])
dense_size=128
num_cat=len(cat_to_id)
batch_size=16
epochs=7
num_classes=len(cat_to_id)


# Create the model
model_glove = tf.keras.models.Sequential()
model_glove.add(tf.keras.layers.Embedding(input_dim=num_words,
                                          output_dim=embedding_dim,
                                          input_length=max_len,
                                          weights=[embedding_matrix]
                                       ))

#model_glove.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=128)))
model_glove.add(tf.keras.layers.LSTM(units=128))
model_rnn.add(tf.keras.layers.Dropout(0.2))
model_glove.add(tf.keras.layers.Dense(num_classes, activation='softmax'))
    

# Compile the model
model_glove.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# Train the model
model_glove.fit(X_train_pad,y_train_id , epochs=epochs, batch_size=batch_size)
model_glove.evaluate(X_test_pad, y_test_id)
Epoch 1/7
9/9 [==============================] - 4s 169ms/step - loss: 3.5065 - accuracy: 0.1714
Epoch 2/7
9/9 [==============================] - 1s 148ms/step - loss: 2.9357 - accuracy: 0.2357
Epoch 3/7
9/9 [==============================] - 1s 152ms/step - loss: 2.5611 - accuracy: 0.2929
Epoch 4/7
9/9 [==============================] - 1s 108ms/step - loss: 2.1017 - accuracy: 0.4286
Epoch 5/7
9/9 [==============================] - 1s 116ms/step - loss: 1.5988 - accuracy: 0.6071
Epoch 6/7
9/9 [==============================] - 1s 88ms/step - loss: 1.0982 - accuracy: 0.7571
Epoch 7/7
9/9 [==============================] - 1s 67ms/step - loss: 0.7189 - accuracy: 0.8786
2/2 [==============================] - 1s 11ms/step - loss: 3.7847 - accuracy: 0.1833



########### MODEL WITH BERT##################

pip install tensorflow keras transformers
from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

#
from tensorflow.keras.preprocessing.sequence import pad_sequences
max_sequence_length=100
# Tokenization and adding special toens
X_train_encoded = [tokenizer.encode(X_train, add_special_tokens=True) for text in X_train]
# Padding
input_ids = pad_sequences(X_train_encoded, maxlen=max_sequence_length, padding='post', truncating='post')
num_classes=len(cat_to_id)
inputs = tf.keras.Input(shape=(max_sequence_length,), dtype=tf.int32)
import tensorflow as tf
from transformers import BertTokenizer, TFBertModel
from tensorflow.keras.layers import Dense
from tensorflow.keras.preprocessing.sequence import pad_sequences

# Define and compile the model
bert_model = TFBertModel.from_pretrained('bert-base-uncased')
inputs = tf.keras.Input(shape=(max_sequence_length,), dtype=tf.int32)
outputs = bert_model(inputs)[1]
outputs = Dense(num_classes, activation='softmax')(outputs)

model = tf.keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

# Train the model
model.fit(x=input_ids, y=y_train_id, epochs=20, batch_size=64)
# For prediction, preprocess the input in the same way
tokenized_inputs_test = [tokenizer.tokenize(text) for text in X_test]
input_ids_test = [tokenizer.convert_tokens_to_ids(tokens) for tokens in tokenized_inputs_test]
input_ids_test = pad_sequences(input_ids_test, maxlen=max_sequence_length, padding='post', truncating='post')
# Evaluate the model
loss, accuracy = model.evaluate(x=input_ids_test, y=y_test_id)

Epoch 1/20
3/3 [==============================] - 3s 947ms/step - loss: 3.2514 - accuracy: 0.2062
Epoch 2/20
3/3 [==============================] - 3s 953ms/step - loss: 3.2550 - accuracy: 0.2062
Epoch 3/20
3/3 [==============================] - 3s 950ms/step - loss: 3.2695 - accuracy: 0.2062
Epoch 4/20
3/3 [==============================] - 3s 957ms/step - loss: 3.2598 - accuracy: 0.2062
Epoch 5/20
3/3 [==============================] - 3s 958ms/step - loss: 3.2604 - accuracy: 0.2062
Epoch 6/20
3/3 [==============================] - 3s 953ms/step - loss: 3.2649 - accuracy: 0.2062
Epoch 7/20
3/3 [==============================] - 3s 948ms/step - loss: 3.2507 - accuracy: 0.2062
Epoch 8/20
3/3 [==============================] - 3s 940ms/step - loss: 3.2564 - accuracy: 0.2062
Epoch 9/20
3/3 [==============================] - 3s 932ms/step - loss: 3.2727 - accuracy: 0.2062
Epoch 10/20
3/3 [==============================] - 3s 944ms/step - loss: 3.2611 - accuracy: 0.2062
Epoch 11/20
3/3 [==============================] - 3s 930ms/step - loss: 3.2527 - accuracy: 0.2062
Epoch 12/20
3/3 [==============================] - 3s 923ms/step - loss: 3.2578 - accuracy: 0.2062
Epoch 13/20
3/3 [==============================] - 3s 921ms/step - loss: 3.2626 - accuracy: 0.2062
Epoch 14/20
3/3 [==============================] - 3s 935ms/step - loss: 3.2546 - accuracy: 0.2062
Epoch 15/20
3/3 [==============================] - 3s 922ms/step - loss: 3.2617 - accuracy: 0.2062
Epoch 16/20
3/3 [==============================] - 3s 918ms/step - loss: 3.2577 - accuracy: 0.2062
Epoch 17/20
3/3 [==============================] - 3s 922ms/step - loss: 3.2602 - accuracy: 0.2062
Epoch 18/20
3/3 [==============================] - 3s 921ms/step - loss: 3.2617 - accuracy: 0.2062
Epoch 19/20
3/3 [==============================] - 3s 929ms/step - loss: 3.2513 - accuracy: 0.2062
Epoch 20/20
3/3 [==============================] - 3s 919ms/step - loss: 3.2497 - accuracy: 0.2062
$\endgroup$
2
  • $\begingroup$ I can't acess the data (says: Forbidden (#403)): how many classes on the y variable? $\endgroup$
    – Memristor
    Jun 15, 2023 at 10:29
  • $\begingroup$ It is a subset of my data, around 30 categories. The whole dataset have around 90 categories $\endgroup$ Jun 18, 2023 at 18:56

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.