1
$\begingroup$

I trained NLP models. This is a subset (200 instances) of my data set of 10,000 instances:This the link of the dataset on pastebin

I compare an LSTM model with a glove model and a BERT model. I expected a good performance with BERT. I can't get past 20% accuracy with BERT at all. I wonder what I'm missing in its implementation.

!pip list
#tensorflow 2.12.0
!python --version
#python 3.10.11

import json 
import tensorflow as tf
import numpy as np
from hyperopt import Trials, STATUS_OK, tpe
from sklearn.model_selection import train_test_split
from keras.layers import Input
from sklearn.metrics import accuracy_score
import pandas as pd

# Reading of file
f = open ('sampled_data.json', "r")
data = json.loads(f.read())

Data preprocessing

X=[x["title"].lower() for x in data]
y=[x["categories"][0].lower() for x in data]
X_train,X_test,y_train,y_test= train_test_split(X,y, test_size=0.2, 
                                                random_state=42)

target preprocessing. To consider the category unknown if not seen in test set

cat_to_id={'<UNK>':'0'}
for cat in y_train:
  if cat not in cat_to_id:
    cat_to_id[cat]=len(cat_to_id)

#MAPPING WITH RESPECT TO THE TRAINING SET
id_to_cat={v:k for k,v in cat_to_id.items()}
def preprocess_Y(Y,cat_to_id):
  res=[]
  for ex in Y:
    if ex not in cat_to_id.keys():
      res.append(cat_to_id['<UNK>'])
    else: 
      res.append(cat_to_id[ex])
  return np.array(res)

y_train_id=preprocess_Y(y_train,cat_to_id)
y_test_id=preprocess_Y(y_test,cat_to_id)
y_test_id=y_test_id.astype(float)

# Tokenization of of features
tokenizer=tf.keras.preprocessing.text.Tokenizer(num_words=10000)
tokenizer.fit_on_texts(X_train)

# TEXT TO SEQUENCE
X_train_seq=tokenizer.texts_to_sequences(X_train)
X_test_seq=tokenizer.texts_to_sequences(X_test)

#PADDING pad_sequences function transform in array
max_len=max([len(length) for length in X_train_seq])
X_train_pad= tf.keras.preprocessing.sequence.pad_sequences(X_train_seq,maxlen=max_len, truncating='post')
X_test_pad= tf.keras.preprocessing.sequence.pad_sequences(X_test_seq,maxlen=max_len, truncating='post')


####### RECCURRENT NEURAL NETWORK###############

vocab_size=len(tokenizer.word_index)
Embed_dim=300
dropout=0.2
dense_size=128
num_cat=len(cat_to_id)
batch_size=16
epochs=15

### CREER LE MODELE
model_rnn=tf.keras.models.Sequential()

# Add an embedding layer
model_rnn.add(tf.keras.layers.Embedding(input_dim=vocab_size, 
                                        output_dim=Embed_dim, 
                                        input_length=max_len))

# Add an LSTM layer
model_rnn.add(tf.keras.layers.LSTM(units=128))
model_rnn.add(tf.keras.layers.Dropout(0.4))

# Dense + activation
model_rnn.add(tf.keras.layers.Dense(units=dense_size,activation='relu'))
#Classifieur + activation
model_rnn.add(tf.keras.layers.Dense(units=num_cat,activation='softmax'))
print(model_rnn.summary())

model_rnn.compile(loss= 'sparse_categorical_crossentropy',
                  optimizer='adam',
                  metrics='accuracy')

model_rnn.fit(X_train_pad,y_train_id, batch_size=batch_size, epochs=epochs)

model_rnn.evaluate(X_test_pad, y_test_id)


Epoch 1/15
9/9 [==============================] - 5s 166ms/step - loss: 3.6699 - accuracy: 0.1643
Epoch 2/15
9/9 [==============================] - 1s 128ms/step - loss: 3.3861 - accuracy: 0.2286
Epoch 3/15
9/9 [==============================] - 1s 157ms/step - loss: 3.1313 - accuracy: 0.2357
Epoch 4/15
9/9 [==============================] - 1s 88ms/step - loss: 3.0774 - accuracy: 0.2286
Epoch 5/15
9/9 [==============================] - 1s 127ms/step - loss: 3.0358 - accuracy: 0.2286
Epoch 6/15
9/9 [==============================] - 0s 27ms/step - loss: 2.9461 - accuracy: 0.2286
Epoch 7/15
9/9 [==============================] - 0s 27ms/step - loss: 2.7970 - accuracy: 0.2357
Epoch 8/15
9/9 [==============================] - 1s 75ms/step - loss: 2.5048 - accuracy: 0.2429
Epoch 9/15
9/9 [==============================] - 1s 86ms/step - loss: 2.2543 - accuracy: 0.3357
Epoch 10/15
9/9 [==============================] - 1s 47ms/step - loss: 1.9985 - accuracy: 0.4357
Epoch 11/15
9/9 [==============================] - 0s 39ms/step - loss: 1.7728 - accuracy: 0.4929
Epoch 12/15
9/9 [==============================] - 1s 41ms/step - loss: 1.5552 - accuracy: 0.5929
Epoch 13/15
9/9 [==============================] - 0s 11ms/step - loss: 1.3320 - accuracy: 0.5929
Epoch 14/15
9/9 [==============================] - 0s 11ms/step - loss: 1.1506 - accuracy: 0.6786
Epoch 15/15
9/9 [==============================] - 0s 42ms/step - loss: 0.9498 - accuracy: 0.7714
2/2 [==============================] - 1s 13ms/step - loss: 6.6335 - accuracy: 0.2000



############# MODEL WITH GLOVE###################
embeddings_index = {}
f = open('glove.6B.300d.txt', encoding='utf-8')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

# Create embedding matrix
word_index=tokenizer.word_index
num_words = len(word_index) + 1
embedding_dim = 300
embedding_matrix = np.zeros((num_words, embedding_dim))
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector


    num_words=len(tokenizer.word_index)+1
embedding_dim=300
max_len=max([len(length) for length in X_train_seq])
dense_size=128
num_cat=len(cat_to_id)
batch_size=16
epochs=7
num_classes=len(cat_to_id)


# Create the model
model_glove = tf.keras.models.Sequential()
model_glove.add(tf.keras.layers.Embedding(input_dim=num_words,
                                          output_dim=embedding_dim,
                                          input_length=max_len,
                                          weights=[embedding_matrix]
                                       ))

#model_glove.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=128)))
model_glove.add(tf.keras.layers.LSTM(units=128))
model_rnn.add(tf.keras.layers.Dropout(0.2))
model_glove.add(tf.keras.layers.Dense(num_classes, activation='softmax'))
    

# Compile the model
model_glove.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# Train the model
model_glove.fit(X_train_pad,y_train_id , epochs=epochs, batch_size=batch_size)
model_glove.evaluate(X_test_pad, y_test_id)
Epoch 1/7
9/9 [==============================] - 4s 169ms/step - loss: 3.5065 - accuracy: 0.1714
Epoch 2/7
9/9 [==============================] - 1s 148ms/step - loss: 2.9357 - accuracy: 0.2357
Epoch 3/7
9/9 [==============================] - 1s 152ms/step - loss: 2.5611 - accuracy: 0.2929
Epoch 4/7
9/9 [==============================] - 1s 108ms/step - loss: 2.1017 - accuracy: 0.4286
Epoch 5/7
9/9 [==============================] - 1s 116ms/step - loss: 1.5988 - accuracy: 0.6071
Epoch 6/7
9/9 [==============================] - 1s 88ms/step - loss: 1.0982 - accuracy: 0.7571
Epoch 7/7
9/9 [==============================] - 1s 67ms/step - loss: 0.7189 - accuracy: 0.8786
2/2 [==============================] - 1s 11ms/step - loss: 3.7847 - accuracy: 0.1833



########### MODEL WITH BERT##################

pip install tensorflow keras transformers
from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

#
from tensorflow.keras.preprocessing.sequence import pad_sequences
max_sequence_length=100
# Tokenization and adding special toens
X_train_encoded = [tokenizer.encode(X_train, add_special_tokens=True) for text in X_train]
# Padding
input_ids = pad_sequences(X_train_encoded, maxlen=max_sequence_length, padding='post', truncating='post')
num_classes=len(cat_to_id)
inputs = tf.keras.Input(shape=(max_sequence_length,), dtype=tf.int32)
import tensorflow as tf
from transformers import BertTokenizer, TFBertModel
from tensorflow.keras.layers import Dense
from tensorflow.keras.preprocessing.sequence import pad_sequences

# Define and compile the model
bert_model = TFBertModel.from_pretrained('bert-base-uncased')
inputs = tf.keras.Input(shape=(max_sequence_length,), dtype=tf.int32)
outputs = bert_model(inputs)[1]
outputs = Dense(num_classes, activation='softmax')(outputs)

model = tf.keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

# Train the model
model.fit(x=input_ids, y=y_train_id, epochs=20, batch_size=64)
# For prediction, preprocess the input in the same way
tokenized_inputs_test = [tokenizer.tokenize(text) for text in X_test]
input_ids_test = [tokenizer.convert_tokens_to_ids(tokens) for tokens in tokenized_inputs_test]
input_ids_test = pad_sequences(input_ids_test, maxlen=max_sequence_length, padding='post', truncating='post')
# Evaluate the model
loss, accuracy = model.evaluate(x=input_ids_test, y=y_test_id)

Epoch 1/20
3/3 [==============================] - 3s 947ms/step - loss: 3.2514 - accuracy: 0.2062
Epoch 2/20
3/3 [==============================] - 3s 953ms/step - loss: 3.2550 - accuracy: 0.2062
Epoch 3/20
3/3 [==============================] - 3s 950ms/step - loss: 3.2695 - accuracy: 0.2062
Epoch 4/20
3/3 [==============================] - 3s 957ms/step - loss: 3.2598 - accuracy: 0.2062
Epoch 5/20
3/3 [==============================] - 3s 958ms/step - loss: 3.2604 - accuracy: 0.2062
Epoch 6/20
3/3 [==============================] - 3s 953ms/step - loss: 3.2649 - accuracy: 0.2062
Epoch 7/20
3/3 [==============================] - 3s 948ms/step - loss: 3.2507 - accuracy: 0.2062
Epoch 8/20
3/3 [==============================] - 3s 940ms/step - loss: 3.2564 - accuracy: 0.2062
Epoch 9/20
3/3 [==============================] - 3s 932ms/step - loss: 3.2727 - accuracy: 0.2062
Epoch 10/20
3/3 [==============================] - 3s 944ms/step - loss: 3.2611 - accuracy: 0.2062
Epoch 11/20
3/3 [==============================] - 3s 930ms/step - loss: 3.2527 - accuracy: 0.2062
Epoch 12/20
3/3 [==============================] - 3s 923ms/step - loss: 3.2578 - accuracy: 0.2062
Epoch 13/20
3/3 [==============================] - 3s 921ms/step - loss: 3.2626 - accuracy: 0.2062
Epoch 14/20
3/3 [==============================] - 3s 935ms/step - loss: 3.2546 - accuracy: 0.2062
Epoch 15/20
3/3 [==============================] - 3s 922ms/step - loss: 3.2617 - accuracy: 0.2062
Epoch 16/20
3/3 [==============================] - 3s 918ms/step - loss: 3.2577 - accuracy: 0.2062
Epoch 17/20
3/3 [==============================] - 3s 922ms/step - loss: 3.2602 - accuracy: 0.2062
Epoch 18/20
3/3 [==============================] - 3s 921ms/step - loss: 3.2617 - accuracy: 0.2062
Epoch 19/20
3/3 [==============================] - 3s 929ms/step - loss: 3.2513 - accuracy: 0.2062
Epoch 20/20
3/3 [==============================] - 3s 919ms/step - loss: 3.2497 - accuracy: 0.2062
Improve this question
$\endgroup$
2
  • $\begingroup$ I can't acess the data (says: Forbidden (#403)): how many classes on the y variable? $\endgroup$
    – Memristor
    Jun 15 at 10:29
  • $\begingroup$ It is a subset of my data, around 30 categories. The whole dataset have around 90 categories $\endgroup$
    – Seydou GORO
    Jun 18 at 18:56

0

Reset to default

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.