Implementing XGBoost with CNN

Question

I am trying to implement XGBoost as a classifier for a pre-trained CNN. The model produces an F1 score of 93, however, when classifying with XGBoost (or with SVM), the F1 drops to 33. It seems to be an issue when fitting the classifier with the training data.

train_dir = os.path.join(main_dir, 'train')
test_dir = os.path.join(main_dir, 'test')

train_data = tf.keras.preprocessing.image_dataset_from_directory(train_dir,
                                                                labels='inferred',
                                                                image_size=img_size,
                                                                batch_size=batch_no,
                                                                seed=seed_no,                                                               
                                                                shuffle=True,
                                                                subset="training",
                                                                validation_split=0.2,
                                                                label_mode="categorical")

validation_data = tf.keras.preprocessing.image_dataset_from_directory(train_dir,
                                                                      labels='inferred',
                                                                      image_size=img_size,
                                                                      batch_size=batch_no,
                                                                      seed=seed_no,
                                                                      shuffle=False,
                                                                      validation_split=0.2,
                                                                      subset="validation",
                                                                      label_mode="categorical")
                                                                      
test_data = tf.keras.preprocessing.image_dataset_from_directory(test_dir,
                                                                labels='inferred',
                                                                image_size=img_size,
                                                                batch_size=batch_no,
                                                                shuffle=False,
                                                                seed=seed_no,
                                                                label_mode="categorical")

Load model and set at output layer

model1 = keras.models.load_model('/.h5')
model1 = Model(inputs=model1.inputs,
                outputs=model1.outputs,
                name='EffNetB2')

effNet_model = Model(model1.input, model1.get_layer('output_layer').output)

Get data labels (test: 60 / train: 192)

test_set_labels =  np.array([])
for x, y in test_data:
  test_set_labels = np.concatenate([test_set_labels, np.argmax(y.numpy(), axis=-1)])

train_set_labels =  np.array([])
for x, y in train_data:
  train_set_labels = np.concatenate([train_set_labels, np.argmax(y.numpy(), axis=-1)])

Use model to extract features

X_train_features = effNet_model.predict(train_data)
X_test_features = effNet_model.predict(test_data)

Test without new classifier (F1: 93)

prediction2 = np.array(list(map(lambda x: np.argmax(x), X_test_features)))
print(classification_report(test_set_labels, prediction2))

Try with XGBoost (F1: 33)

from xgboost import XGBClassifier

xgb = XGBClassifier(objective='multiclass:softmax', learning_rate = 0.1,
              max_depth = 15, n_estimators = 500)

xgb.fit(X_train_features, train_set_labels)

test_pred = xgb.predict(X_test_features)

predictionsXGB = np.array(list(map(lambda x: np.argmax(x), test_pred)))
print(classification_report(test_set_labels, predictionsXGB))
```