I have a classification neural network and nominal input data on which it is trained, however the input data has for each feature a systematic (up and down) uncertainty. How should the accuracy of the classifier be qualified and visualised using these input data uncertainties? I have a simple MWE example composed using the iris dataset; the intention is that is should be copy-pastable easily into a Jupyter notebook.

Lotsa imports:

import numpy as np
import datetime
from IPython.display import SVG
from keras.datasets import mnist
from keras import activations
from keras import backend as K
from keras.layers import Dense, Input, concatenate, Conv1D, Conv2D, Dropout, MaxPooling1D, MaxPooling2D
from keras.layers import Dense, Flatten
from keras.models import Model, Sequential, load_model
from keras.utils import plot_model
from keras.utils.vis_utils import model_to_dot
from matplotlib import gridspec
from matplotlib.ticker import NullFormatter, NullLocator, MultipleLocator
from scipy import stats
from sklearn.datasets import load_iris
from sklearn.metrics import auc, roc_curve
from sklearn.model_selection import train_test_split
from vis.utils import utils
from vis.visualization import visualize_activation
from vis.visualization import visualize_saliency
import datetime
import keras
import matplotlib.pylab as plt
import pandas as pd
import random
import seaborn as sns
import talos as ta
import warnings
%matplotlib inline
plt.rcParams['figure.figsize'] = [10, 10]

Let's load the iris dataset and limit it to two classes, then prepare it for training.

iris = load_iris()
df = pd.DataFrame(
    data    = np.c_[iris['data'], iris['target']],
    columns = iris['feature_names'] + ['target']
df = df.query('target != 2')

df['labels'] = df['target'].astype('category').cat.codes
x = df[['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']]
y = df['target']
# Convert class vectors to binary class matrices using 1 hot encoding.
# 0 ---> 1, 0, 0
# 1 ---> 0, 1, 0
# 2 ---> 0, 0, 1
num_classes = len(y.unique())
y = keras.utils.to_categorical(y, len(y.unique()))

x = np.asarray(x)
y = np.asarray(y)

x = x.reshape(len(x), 4, 1)

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.33, shuffle = True)

Let's make some simple model for classification.

model = Sequential()
model.add(Dense(5, input_shape = (4, 1),         activation = 'tanh'))
model.add(Dense(5,                               activation = 'tanh'))
model.add(Dense(num_classes,                     activation = 'softmax', name = 'preds'))
model.compile(loss = "categorical_crossentropy", optimizer  = "nadam", metrics = ['accuracy'])
SVG(model_to_dot(model).create(prog='dot', format='svg'))

Now for a quick bit of training...

def model_evaluation(model, x_test, y_test, verbose=False):
    score = model.evaluate(x_test, y_test, verbose=verbose)
    print('max. test accuracy observed:', max(model.history.history['val_acc']))
    print('max. test accuracy history index:', model.history.history['val_acc'].index(max(model.history.history['val_acc'])))
    plt.legend(['train_accuracy', 'test_accuracy'], loc='best')
    batch_size      = 2,
    epochs          = 100,
    verbose         = False,
    validation_data = (x_test, y_test),
model_evaluation(model, x_test, y_test, verbose=False)

Now, let's add some uncertainties for each of the features:

for column in ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']:
    uncertainties_up   = 0.1 * df[column].mean() * np.random.random_sample(size=(len(df)))
    uncertainties_down = df[column].mean() * np.random.random_sample(size=(len(df)))
    df[column + " uncertainty up"] = df[column] + uncertainties_up

And now what actually comes next, in order to qualify the classifier given these various input data uncertainties?


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.