I'm trying to analyze the performance of a binary classifier on the test set on different ranges of the predictions. the classifier has a .97 ROC AUC on the test. Then I binarize the test set predictions into bins to check the ROC AUC on every bucket but in the bins, it has very low performance.
Reproducible example:
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
n = 50
X, y = make_classification(n_samples = 10000,n_features = n, n_informative=27,n_classes = 2, random_state = 42)
X = pd.DataFrame(X, columns = [f"x_{i}" for i in range(n)])
y = pd.DataFrame(y, columns = ["target"])
frame = pd.merge(X,y, left_index = True, right_index = True)
frame.loc[:,"split"] = np.random.choice(a = ["train","test"], p = [.7,.3], size = frame.shape[0])
train_df = frame.query("split == 'train'").drop("split", axis = 1)
test_df = frame.query("split == 'test'").drop("split", axis = 1)
clf = RandomForestClassifier(random_state = 42).fit(train_df.drop("target", axis = 1),train_df["target"])
preds = clf.predict_proba(test_df.drop("target", axis = 1))[:,1]
roc_ = roc_auc_score(y_true = test_df["target"], y_score = preds)
print(f"ROC AUC: {roc_}")
test_df.loc[:,"prediction"] = clf.predict_proba(test_df.drop("target", axis = 1))[:,1]
test_df.loc[:,"band"] = pd.qcut(q = 10, x = test_df.prediction, duplicates = "drop")
def get_auc_(y_true, y_score):
try:
return roc_auc_score(y_true = y_true, y_score = y_score)
except:
return np.NaN
test_df.groupby("band").apply(lambda x: get_auc_(y_true = x["target"], y_score = x["prediction"]))
band
(0.009000000000000001, 0.16] NaN
(0.16, 0.23] 0.051780
(0.23, 0.3] 0.592401
(0.3, 0.39] 0.633804
(0.39, 0.5] 0.626548
(0.5, 0.61] 0.629141
(0.61, 0.7] 0.633596
(0.7, 0.77] 0.702138
(0.77, 0.84] 0.477372
(0.84, 0.98] 0.480072
dtype: float64
My question is what explains this low score in the different bins?