0
$\begingroup$

I am trying to optimize the below mentioned hyper-parameters of MLP with range as follows.

  1. Number of hidden layers (n): 1-10
  2. Number of perceptrons (p): 25, 50, 75, ..., 200
  3. Activation function: Identity, Logistic, Tanh, Relu
  4. Initial learning rate: 0.01, 0.02, 0.03, ..., 1

The SKlearn based MLP uses a tuple as an input for the hidden_layer_sizes. I want to directly optimize both the number of layers (n) and the number of perceptrons (p) in each layer.

That is at the end, I want specific values of above 4 mentioned hyperparameters.

  1. How do I impose the correct search space to reflect various combinations of n and p being picked randomly by the optimizer?

  2. Is there way to graphically present the improvement in the model over different search iterations?

Here is my code and the input dataframe for this code can be found here. Please help me. Any help would be appreciated. Thank you in advance!

### Load dependencies
import pandas as pd
import numpy as np

### Dependencies for MLP and cross validation
from sklearn.neural_network import MLPRegressor
from sklearn.metrics import mean_squared_error as mse
from sklearn.model_selection import cross_val_score

### Dependencies for printing results
import matplotlib.pyplot as plt

### Dependencies for Bayesian optimization
from hyperopt import hp
from hyperopt import STATUS_OK
from hyperopt import tpe# Algorithm
from hyperopt import Trials
from hyperopt import fmin

from datetime import datetime

start_time = datetime.now()

### Load dataframe and top principal components
data = pd.read_csv (r'C:\Users\mishra2\Desktop\PCA + FFN\data.csv')
most_important_names = ['11939', '13826']

###============================================================================================###
### Define MLP Hyper-parameters
n = 3 # ----------- Number of layers
p = 50 # ---------- Number of perceptrons in each layer
i = 1 # ------------ Initializing iteration count
MLPsize = []

###============================================================================================###
### While loop to create tuple for MLP hyperparameter input
while i <= n:
    MLPsize.append(p)
    i += 1

###============================================================================================###
### Define sklearn model parameters
Model = MLPRegressor(hidden_layer_sizes=tuple(MLPsize), activation='logistic', 
                     solver='adam', batch_size='auto', learning_rate='adaptive',
                     learning_rate_init=0.001, max_iter=5000, shuffle=False)

###============================================================================================###
### Define function to run train-test simulation

msk = np.random.rand(len(data)) < 0.8
Train = data[msk].reset_index(drop=True)
Test = data[~msk].reset_index(drop=True)

def RunModel (h):

    global k1, count_train, count_test, Train_pred, Test_pred, Train_true, Test_true, Train_error, Test_error
    count_train = []
    Train_pred, Test_pred, Train_true, Test_true, Train_error, Test_error = [],[],[],[],[],[]
    step = 1

    for k1 in range(1, len(Train)+1, step):
        count_train.append(k1)
        Model.fit(Train.iloc[:k1,1:7], Train[h][:k1])
        Train_pred = Model.predict(Train.iloc[:k1,1:7])
        Train_error.append(mse(Train[h][:k1], Train_pred))
        Train_true = np.array(Train[h]).reshape(-1, 1)
        Train_pred = Train_pred.reshape(-1, 1)
        print('R2 score for train set is:', Model.score(Train.iloc[:,1:7], Train_true))
        plt.plot(Train["Storm ID"][:k1], Train_true[:k1], color='black', label='True Value for Train Set', linestyle='solid', linewidth=0.5, marker=None, markersize=0)
        plt.plot(Train["Storm ID"][:k1], Train_pred, color='red', label='Pred. Value for Train Set', linestyle='dashed', linewidth=0.5, marker=None, markersize=0)
        plt.legend()
        plt.xlabel('Storm ID number')
        plt.ylabel('Surge values in metres')
        plt.show()


    Test_pred = Model.predict(Test.iloc[:,1:7])
    Test_error.append(mse(Test[h], Test_pred))
    Test_true = np.array(Test[h]).reshape(-1, 1)
    Test_pred = Test_pred.reshape(-1, 1)
    print('R2 score for test set is:', Model.score(Test.iloc[:,1:7], Test_true))
    plt.plot(Test["Storm ID"], Test_true, color='black', label='True Value for Test Set', linestyle='solid', linewidth=0.5, marker=None, markersize=0)
    plt.plot(Test["Storm ID"], Test_pred, color='red', label='Pred. Value for Test Set', linestyle='dashed', linewidth=0.5, marker=None, markersize=0)
    plt.legend()
    plt.xlabel('Storm ID number')
    plt.ylabel('Surge values in metres')
    plt.show()

###============================================================================================###
### Define objective function

def objective(self):

    # Perform K-fold cross validation
    print(cross_val_score(Model, Train.iloc[:,1:7], Train[h], scoring='r2', cv=10))

    # Loss to be minimized
    loss = 1 - Model.score(Test.iloc[:,1:7], Test_true)

    return {'loss': loss, 'number of hidden_layers': n, 'number of perceptrons': p,
            'status': STATUS_OK}

###============================================================================================###    
### Define the search space
def MLP_size(n, p):
    i = 1
    n, p = int(n), int(p)
    MLP_size = []
    while i <= n:
        MLP_size.append(p)
        i += 1
    return tuple(MLP_size)

hidden_layer_sizes=tuple(MLPsize)
space = {
    'activation': hp.choice('activation', ['logistic', 'tanh', 'relu']),
    'learning_rate_init': hp.uniform('learning_rate_init', 0.001, 1),
    'n': hp.quniform('num_hidden_layers', 1, 20, 1),
    'p': hp.quniform('num_perceptrons', 25, 500, 25),
    'hidden_layer_sizes': MLP_size(n, p)
}

### Define algorithm for surrogate model
tpe_algorithm = tpe.suggest

### Trials object to track progress
bayes_trials = Trials()

###============================================================================================###
### Define optimisation function    

def BayOpt():
    MAX_EVALS = 50

    global best
    best = fmin(fn = objective, space = space, algo = tpe.suggest, 
                max_evals = MAX_EVALS, trials = bayes_trials)
###============================================================================================###

h = '11939'
RunModel(h)
print('Validation score in model is:', Model.score(Test.iloc[:,1:7], Test_true))
print('MSE of model for test set is:', mse(Test_true, Test_pred))
BayOpt()
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.