0
$\begingroup$

I have trained a LSTM model to detect fake domain names.

My dataset is like this:

domain,fake
google, 0
bezqcuoqzcjloc,1
...

with 50% normal and 50% fake domain names.

Here's my code to train the LSTM:

def build_model(max_features, maxlen):
    """Build LSTM model"""
    model = Sequential()
    model.add(Embedding(max_features, 128, input_length=maxlen))
    model.add(LSTM(128))
    model.add(Dropout(0.5))
    model.add(Dense(1))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer='rmsprop')

    return model

def run(max_epoch=25, nfolds=10, batch_size=128):
    """Run train/test on logistic regression model"""
    indata = data.get_data()

    # Extract data and labels
    X = [x[1] for x in indata]
    labels = [x[0] for x in indata]

    # Generate a dictionary of valid characters
    valid_chars = {x:idx+1 for idx, x in enumerate(set(''.join(X)))}

    max_features = len(valid_chars) + 1
    maxlen = 100

    # Convert characters to int and pad
    X = [[valid_chars[y] for y in x] for x in X]
    X = sequence.pad_sequences(X, maxlen=maxlen)

    # Convert labels to 0-1
    y = [0 if x == 'benign' else 1 for x in labels]

    final_data = []

    for fold in range(nfolds):
        print("fold %u/%u" % (fold+1, nfolds))
        X_train, X_test, y_train, y_test, _, label_test = train_test_split(X, y, labels, 
                                                                           test_size=0.2)

        print("Build model...")
        model = build_model(max_features, maxlen)

        print("Train...")
        X_train, X_holdout, y_train, y_holdout = train_test_split(X_train, y_train, test_size=0.05)
        best_iter = -1
        best_auc = 0.0
        out_data = {}

        for ep in range(max_epoch):
            model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=1)

            t_probs = model.predict_proba(X_holdout)
            t_auc = sklearn.metrics.roc_auc_score(y_holdout, t_probs)

            print("Epoch %d: auc = %f (best=%f)" % (ep, t_auc, best_auc))

            if t_auc > best_auc:
                best_auc = t_auc
                best_iter = ep

                probs = model.predict_proba(X_test)

                out_data = {'y':y_test, 'labels': label_test, 'probs':probs, 'epochs': ep,
                            'confusion_matrix': sklearn.metrics.confusion_matrix(y_test, probs > .5)}

                print(sklearn.metrics.confusion_matrix(y_test, probs > .5))
            else:
                # No longer improving...break and calc statistics
                if (ep-best_iter) > 2:
                    break
        print('Saving LSTM model...')
        model.save('LSTMmodel.h5')

        final_data.append(out_data)

    return final_data


if __name__ == '__main__':
    run()

I trained and tested it on dataset n°1.

Then I decided to see what were the predictions using this trained model on another dataset, similar to dataset n°1 but with different domain names obviously.

Here's my code:

LSTM_model = load_model('LSTMmodel_2.h5')
data = pickle.load(open('testdata.pkl', 'rb'))

# Extract data and labels
X = [x[1] for x in data]
labels = [x[0] for x in data]

# Generate a dictionary of valid characters
valid_chars = {x:idx+1 for idx, x in enumerate(set(''.join(X)))}

max_features = len(valid_chars) + 1
maxlen = 100

# Convert characters to int and pad
X = [[valid_chars[y] for y in x] for x in X]
X = sequence.pad_sequences(X, maxlen=maxlen)

# Convert labels to 0-1
y = [0 if x == 'benign' else 1 for x in labels]


y_pred = LSTM_model.predict(X)

acc = accuracy_score(y, y_pred.round())

print(acc)

print(sklearn.metrics.confusion_matrix(y, y_pred.round()))

print(sklearn.metrics.f1_score(y, y_pred.round()))

But as results for accuracy, confusion matrix or f1-score I get these:

accuracy = 0.541563570018245
confusion matrix = [[26764  3258]
                    [33427 16573]]
F1-score = 0.47466025117784355

What is wrong with my model? Should I do several epochs when testing the trained model on the dataset n°2?

What I don't get is that when I trained and test my model I have very good results and when checking on a new dataset it does not perform as well.

| improve this question | | | | |
$\endgroup$
0
$\begingroup$

If you model converges on your training data but does not work on test data then the likely case if that you have over-fit your model. Try running until max_epoch=5, and then save model to see if it works better.

| improve this answer | | | | |
$\endgroup$
  • $\begingroup$ I tried but it does not improve the performance very much. I increased batch size to 640 and tried sgd optimizer, it is a bit better : accuracy = 0.6373247356976831 confusion matrix = [[12510 17512] [11510 38490]] F1-score = 0.7262127129676799 But still not so good. $\endgroup$ – Laure D Aug 30 '18 at 8:50

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.