Keras LSTM model not performant

Question

I have trained a LSTM model to detect fake domain names.

My dataset is like this:

domain,fake
google, 0
bezqcuoqzcjloc,1
...

with 50% normal and 50% fake domain names.

Here's my code to train the LSTM:

def build_model(max_features, maxlen):
    """Build LSTM model"""
    model = Sequential()
    model.add(Embedding(max_features, 128, input_length=maxlen))
    model.add(LSTM(128))
    model.add(Dropout(0.5))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer='rmsprop')

    return model

def run(max_epoch=25, nfolds=10, batch_size=128):
    """Run train/test on logistic regression model"""
    indata = data.get_data()

    # Extract data and labels
    X = [x[1] for x in indata]
    labels = [x[0] for x in indata]

    # Generate a dictionary of valid characters
    valid_chars = {x:idx+1 for idx, x in enumerate(set(''.join(X)))}

    max_features = len(valid_chars) + 1
    maxlen = 100

    # Convert characters to int and pad
    X = [[valid_chars[y] for y in x] for x in X]
    X = sequence.pad_sequences(X, maxlen=maxlen)

    # Convert labels to 0-1
    y = [0 if x == 'benign' else 1 for x in labels]

    final_data = []

    for fold in range(nfolds):
        print("fold %u/%u" % (fold+1, nfolds))
        X_train, X_test, y_train, y_test, _, label_test = train_test_split(X, y, labels, 
                                                                           test_size=0.2)

        print("Build model...")
        model = build_model(max_features, maxlen)

        print("Train...")
        X_train, X_holdout, y_train, y_holdout = train_test_split(X_train, y_train, test_size=0.05)
        best_iter = -1
        best_auc = 0.0
        out_data = {}

        for ep in range(max_epoch):
            model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=1)

            t_probs = model.predict_proba(X_holdout)
            t_auc = sklearn.metrics.roc_auc_score(y_holdout, t_probs)

            print("Epoch %d: auc = %f (best=%f)" % (ep, t_auc, best_auc))

            if t_auc > best_auc:
                best_auc = t_auc
                best_iter = ep

                probs = model.predict_proba(X_test)

                out_data = {'y':y_test, 'labels': label_test, 'probs':probs, 'epochs': ep,
                            'confusion_matrix': sklearn.metrics.confusion_matrix(y_test, probs > .5)}

                print(sklearn.metrics.confusion_matrix(y_test, probs > .5))
            else:
                # No longer improving...break and calc statistics
                if (ep-best_iter) > 2:
                    break
        print('Saving LSTM model...')
        model.save('LSTMmodel.h5')

        final_data.append(out_data)

    return final_data


if __name__ == '__main__':
    run()

I trained and tested it on dataset n°1.

Then I decided to see what were the predictions using this trained model on another dataset, similar to dataset n°1 but with different domain names obviously.

Here's my code:

LSTM_model = load_model('LSTMmodel_2.h5')
data = pickle.load(open('testdata.pkl', 'rb'))

# Extract data and labels
X = [x[1] for x in data]
labels = [x[0] for x in data]

# Generate a dictionary of valid characters
valid_chars = {x:idx+1 for idx, x in enumerate(set(''.join(X)))}

max_features = len(valid_chars) + 1
maxlen = 100

# Convert characters to int and pad
X = [[valid_chars[y] for y in x] for x in X]
X = sequence.pad_sequences(X, maxlen=maxlen)

# Convert labels to 0-1
y = [0 if x == 'benign' else 1 for x in labels]


y_pred = LSTM_model.predict(X)

acc = accuracy_score(y, y_pred.round())

print(acc)

print(sklearn.metrics.confusion_matrix(y, y_pred.round()))

print(sklearn.metrics.f1_score(y, y_pred.round()))

But as results for accuracy, confusion matrix or f1-score I get these:

accuracy = 0.541563570018245
confusion matrix = [[26764  3258]
                    [33427 16573]]
F1-score = 0.47466025117784355

What is wrong with my model? Should I do several epochs when testing the trained model on the dataset n°2?

What I don't get is that when I trained and test my model I have very good results and when checking on a new dataset it does not perform as well.

Answer 1

If you model converges on your training data but does not work on test data then the likely case if that you have over-fit your model. Try running until max_epoch=5, and then save model to see if it works better.