LSTM fot text classification always returns the same results

Question

Hello fellow Data Scientists, I'm trying to make a classifier that was to classify sequences of text into some predefined classes, but i always get the same output, can anyone help me understand why? The training of the model:

   # The maximum number of words to be used. (most frequent)
MAX_NB_WORDS = 100
#2155
# Max number of words in each complaint.
MAX_SEQUENCE_LENGTH = 100
# This is fixed.
EMBEDDING_DIM = 20


cf.go_offline()
cf.set_config_file(offline=False, world_readable=True)

def treina(model_name):

    df = pd.read_csv("divididos.csv",sep='§',header=0)
    df.info()

    max_len = 0
    for value in df.Perguntas:
        if(len(value)>max_len):
            max_len = len(value)
    max_words = 0
    for value in df.Perguntas:
        word_count = len(value.split(" "))
        if(word_count>max_words):
            max_words = word_count

    tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True)
    tokenizer.fit_on_texts(df['Perguntas'].values)
    word_index = tokenizer.word_index
    X = tokenizer.texts_to_sequences(df['Perguntas'].values)
    X = pad_sequences(X, maxlen=MAX_SEQUENCE_LENGTH)
    Y = pd.get_dummies(df['Class']).values

    X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.05, random_state = 42)
    print(X_train.shape,Y_train.shape)
    print(X_test.shape,Y_test.shape)

    #Balance data
    sm = SMOTE(random_state=12)
    X_train, Y_train = sm.fit_sample(X_train, Y_train)
    print(X_train.shape,Y_train.shape)

    #LSTM net
    model = Sequential()
    model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))
    model.add(LSTM(20, dropout=0.2, recurrent_dropout=0.2,activation="relu",return_sequences=True))
    model.add(LSTM(10, dropout=0.2, recurrent_dropout=0.2,activation="relu"))
    model.add(Dropout(0.2))
    model.add(Dense(11, activation='softmax'))
    opt = adam(lr=0.3)
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])

    epochs = 100
    batch_size = 20

    history = model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size,validation_split=0.1)
    accr = model.evaluate(X_test,Y_test)
    print('Test set\n  Loss: {:0.3f}\n  Accuracy: {:0.3f}'.format(accr[0],accr[1]))
    model.save(model_name)
    return model

and the testing:

def corre(modelo):
    labels = ["a","b","c","d","e","f","g","h","i","j","k"]
    model = load_model(modelo)
    a = 0
    tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True)
    while (a==0):
        new_complaint = input()
        new_complaint = [new_complaint]
        seq = tokenizer.texts_to_sequences(new_complaint)
        padded = pad_sequences(seq, maxlen=MAX_SEQUENCE_LENGTH)
        pred = model.predict(padded)
        print(pred, labels[np.argmax(pred)])

Thank you for your time

Answer 1

You are not using the same Tokenizer in testing which you used in training, so texts_to_sequences is not outputting the required result.