my xgboost model accuracy decreases after grid search with [duplicate]

Question

I tried grid search for hyperparameter tuning in XGBoost classifier but the best accuracy is less than the accuracy without any tuning

// this is the code before the grid search
xg_cl = xgb.XGBClassifier(objective='binary:logistic', seed = 22)
xg_cl.fit(x_train, y_train)
y_pred = xg_cl.predict(x_test)
print(metrics.confusion_matrix(y_test, y_pred))
print(metrics.accuracy_score(y_test, y_pred))
[[11  1]
[ 1 26]]
0.9487179487179487

also, worth mentioning that the dataset shape is (195, 11) after PCA, and i am trying to classify whether a patient has a parkinsons disease or not.

// this is the grid search code
clf_xgb = xgb.XGBClassifier(objective = 'binary:logistic')
params__grid = {
    'n_estimators' : range(50,150,10),
    'max_depth': range(2, 12),
    'colsample_bytree': np.arange(0.5,1,0.1),
    'reg_alpha' : np.arange(0,0.6,0.1),
    'reg_lambda' : np.arange(0,0.8,0.1)


}
search = GridSearchCV(estimator=clf_xgb, param_grid=params__grid, scoring = 'accuracy',
                            cv = 4 )
search.fit(x_train,y_train)

print('best score:/n',search.best_score_)
print('bestparams:/n' ,search.best_params_)

best score:
0.9038461538461539

best params:
{'colsample_bytree': 0.5,
     'max_depth': 7,
     'n_estimators': 50,
     'reg_alpha': 0.2,
     'reg_lambda': 0.1}

then I used these parameters to build and train a new classifier

clf_xgb_1 = xgb.XGBClassifier(objective = 'binary:logistic', max_depth = 7, n_estimators = 50, reg_alpha = 0.2 ,
                              reg_lambda = 0.1, colsample_bytree = 0.5 )
clf_xgb_1.fit(x_train,y_train)
y_pred_2 = clf_xgb_1.predict(x_test)
print('accuracy:/n', metrics.accuracy_score(y_test,y_pred_2))
print('confusion matrix:/n', metrics.confusion_matrix(y_test,y_pred_2))

accuracy:
0.8974358974358975

confusion matrix:
array([[ 9,  3],
       [ 1, 26]], dtype=int64)

How come my results are worse? I would expect that GridSearch would improve on the results.

Answer 1

I think there are several points to take into account:

• First, it is possible that, in this case, the default XGBoost hyperparameters are a better combination that the ones your are passing through your params__grid combinations, you could check for it

• Although it does not explain your case, keep in mind that the best_score given by the GridSearchCV object is the Mean cross-validated score of the best_estimator (source), it is, the mean test score across the k trainings over the k-folds, so it can maybe give you a more reliable score value (including also the standard deviation of the k scores)

Consider also that you do not need to retrain your model with the best params fron the grid search CV, since you can access the best model via search.best_model

Lastly, I would also recommend other hyperparameter tuning methods like the bayesian tuning, below a sample code gist using hyperopt (info about bayesian optimization here):

from hyperopt import hp
from sklearn.model_selection import StratifiedKFold

from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
from sklearn.model_selection import cross_val_score

def return_model_scores(params, dataX, dataY, n_folds=5):
    import numpy as np

    defined_model, edp_cost_scores, histories = 
    evaluate_model_cost(dataX = dataX, dataY = dataY)   

return np.array(edp_cost_scores).mean()


param_space = {'max_depth' : hp.choice('max_depth', range(5, 30, 1)),
  'learning_rate' : hp.quniform('learning_rate', 0.01, 0.5, 0.01),
  'n_estimators' : hp.choice('n_estimators', range(20, 205, 5)),
  'gamma' : hp.quniform('gamma', 0, 0.50, 0.01),
  'min_child_weight' : hp.quniform('min_child_weight', 1, 10, 1),
  'subsample' : hp.quniform('subsample', 0.1, 1, 0.01),
  'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1.0, 0.01),
  'scale_pos_weight': hp.uniform('scale_pos_weight', 0.2, 0.8)}

# Some variable 
dataX = X_train
dataY = y_train
assert len(X_train)==len(y_train)
n_folds=5

global best # global variable defined for convenience of this use case
best = 0
i = 0
def f(params):
    cost = return_model_scores(params, dataX, dataY, n_folds=5)  
    if i == 0:
        best = cost

    if cost < best:
        best = cost
    print('new best:', best, params)

    return {'loss': cost, 'status': STATUS_OK}

trials = Trials()
best = fmin(f, param_space, algo=tpe.suggest, max_evals=10, trials=trials)

print ('best:')
print (best)

where evaluate_model_cost is the function you build for getting the cost values after evaluation:

def evaluate_model_cost(dataX, dataY):
    from tqdm import tqdm
    from sklearn.model_selection import KFold

    scores, histories = list(), list()
    # prepare cross validation
    kfold = KFold(10, shuffle=True, random_state=1)
    # enumerate splits
    k = 0
    for train_ix, test_ix in tqdm(kfold.split(dataX)):
        print('kfold {}'.format(k))

        # select rows for train and test
        trainX, trainY, testX, testY = dataX.iloc[train_ix], 
          dataY.iloc[train_ix], dataX.iloc[test_ix], dataY.iloc[test_ix]

        # fit model
        history = defined_model.fit(trainX, trainY, 
                                eval_metric= 'auc',
                                eval_set=[(testX, testY)])                               
    # evaluate model
    y_preds = defined_model.predict(testX[features_to_train_on])

    y_true_values = testY
    true_predicted_tuples = pd.DataFrame({'y_true': y_true_values, 'y_predicted': y_preds, 'days_till_slag': days_until_slag})
    true_predicted_tuples = true_predicted_tuples.reset_index(drop=True)
    #true positives, false positives and false negatives number
    tp_savings = 0

    first_tp_detected = False

    while first_tp_detected==False:
        for index in true_predicted_tuples.index:
            # TRUE POSITIVE condition:
            if ((true_predicted_tuples.iloc[index]['y_true']==1)&(true_predicted_tuples.iloc[index]['y_predicted']==1)):
                tp_savings += return_true_positive_savings(true_predicted_tuples.iloc[index]['days_till_slag'])
            break
        first_tp_detected = True

    fp_number = len(true_predicted_tuples[(true_predicted_tuples.y_true==0)&(true_predicted_tuples.y_predicted==1)])
    fn_number = len(true_predicted_tuples[(true_predicted_tuples.y_true==1)&(true_predicted_tuples.y_predicted==0)])

    final_cost = ((costs_dict['fp_cost'])*fp_number) + 
        ((costs_dict['fn_cost'])*fn_number) - tp_savings

    score = final_cost_custom_function

    print('score en evaluate_model_with_slag_days', score)

    # append scores
    scores.append(score)
    histories.append(history)

    k = k + 1

return defined_model, scores, histories

where final_cost_custom_function is your custom function of costs to minimize. This might be a bit complicated case, if you want a very simple example of bayesian optimization, start with:

import pickle
import time
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials

def objective(x):
    return {
     'loss': x ** 2, 
     'status': STATUS_OK,
     # -- store other results like this
     'eval_time': time.time(),
     'other_stuff': {'type': None, 'value': [0, 1, 2]},
     # -- attachments are handled differently
     'attachments':
        {'time_module': pickle.dumps(time.time)}
     }
trials = Trials()
best = fmin(objective,
 space=hp.uniform('x', -3, 3),
 algo=tpe.suggest,
 max_evals=100,
 trials=trials)

print('with 100 trials: ', best)

trials = Trials()
best = fmin(objective, space=hp.uniform('x', -3, 3), algo=tpe.suggest,
 max_evals=1000, trials=trials)

print('with 1000 trials: ', best)