I am trying to build an image classifier for a set of images containing cats and dogs. I am very new to the dark art of creating Neural Network models. I have had success building models with Keras in the past, and am trying to build some more advanced models using tensorflow.
The input data is 128x128 greyscale images, and labels are one-hot encoded for the two classes (cat/dog = [1,0],[0,1]) I have checked and re-checked my training data, it is balanced etc.
I am however running short iterations (less than 3000) over samples of the data (around 1000 images) because I don't have a GPU, or enough memory to multitask while training on the full set. I understand this is not enough iterations to fully train the model, but I want to see at least some change in the accuracy or loss before I commit to a night's worth of training.
Things to look at: loss function, accuracy metrics, structure of network, training operation, training routine
Core problem: network is not training, loss converges to particular value and sticks, the accuracy remains the same over iterations
The loss converges to the same value and does not change. The final loss value looks like: (for batch size of 10 (= len(loss) here, and the two classes)
loss = [[0.31326172 0.6931472 ], [0.31326172 0.6931472 ], [1.3132617 0.6931472 ], [0.31326172 0.6931472 ], [0.31326172 0.6931472 ], [0.31326172 0.6931472 ], [1.3132617 0.6931472 ], [0.31326172 0.6931472 ], [1.3132617 0.6931472 ], [0.31326172 0.6931472 ]]
Some typical results, over 100 and 1000 iterations:
The loss over a ~30 iterations looks like:
I have tried:
different learning rates (0.1 to 0.001)
different loss functions (sigmoid cross entropy, softmax cross entropy)
different batch sizes (1,10,32)
I feel like I'm making some fundamental error here, can anybody see whats wrong?
The model:
# Helper functions to setup convolution layers
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(input_to_layer, filter_params):
return tf.nn.conv2d(input_to_layer, filter_params, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
def convpool(layer_in, WEIGHT):
conv = conv2d(layer_in, weight_variable(WEIGHT))
conv_bias = conv + bias_variable([WEIGHT[-1]])
conv_bias_relu = tf.nn.relu(conv_bias)
layer_out = max_pool_2x2(conv_bias_relu)
return layer_out
x = tf.placeholder(dtype = tf.float32, shape = [None, 128, 128], name = "Xplaceholder")
y = tf.cast(tf.placeholder(dtype = tf.int32, shape = [None,2], name = "yplaceholder"),tf.float32)
image = tf.reshape(x, [-1,128,128,1])
# LAYERS
layer1_out = convpool(image,[3,3,1,64]) #OUT: -1x64x64x64
layer2_out = convpool(layer1_out, [3,3,64,128]) #OUT: -1x32x32x128
layer3_out = tf.contrib.layers.fully_connected(layer2_out, 512, tf.nn.relu) #OUT: -1x512
layer4_out = tf.nn.dropout(layer3_out, keep_prob=0.99)
layer5_out = tf.reshape(layer4_out, [10, 32*32*512]) #BATCH SIZE HERE IS 10
layer6_out = tf.contrib.layers.fully_connected(layer5_out, 2, tf.nn.softmax) # Logit
loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=layer6_out, labels=y)
train_op = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
correct_pred = tf.one_hot(tf.argmax(layer6_out, 1), 2, axis = 1)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
Training routine:
def batch_trainer(training_set, training_labels, eval_set, eval_labels, iterations = 2500, batch_size = 125):
sess.run(tf.global_variables_initializer())
print("")
print("Training:")
train_accuracy_logs = []
eval_accuracy_logs = []
epochs_completed = 0
index_in_epoch = 0
num_examples = len(training_labels)
assert batch_size <= num_examples # BATCH SIZE MUST BE LESS SIZE OF TRAINING SET!
def next_batch(training_set, training_labels, epochs_completed, index_in_epoch, batch_size):
start = index_in_epoch
index_in_epoch += batch_size
if index_in_epoch > num_examples:
epochs_completed += 1
# re shuffles
perm = np.random.shuffle(np.arange(num_examples))
training_set = training_set[perm]
training_labels = training_labels[perm]
# start next epoch
start = 0
index_in_epoch = batch_size
assert batch_size <= num_examples
end = index_in_epoch
return training_set[start:end], training_labels[start:end]
for i in range(iterations):
batch_xs, batch_ys = next_batch(training_set, training_labels, epochs_completed, index_in_epoch, batch_size)
_, accuracy_val = sess.run([train_op, accuracy], feed_dict={x: batch_xs, y: batch_ys})
train_accuracy_logs.append(accuracy_val)
if i%batch_size == 0:
eval_accuracy_logs.append(test_model(eval_set, eval_labels))
average_acc = (max(train_accuracy_logs)+max(eval_accuracy_logs)+min(train_accuracy_logs)+min(eval_accuracy_logs))/4
print(" " + str(i).zfill(4)+": Accuracy: {:.1f}%".format(accuracy_val*100))
plt.rcParams['figure.figsize'] = (4.0, 1.0)
ax = plt.axes()
plt.plot(train_accuracy_logs, c = 'r', linewidth = 2)
plt.plot(np.arange(0,i+1,batch_size),eval_accuracy_logs, c = 'black', linewidth = 2)
plt.grid()
plt.rc('font',size=20)
plt.text(1.3, average_acc, "{:.2f}%".format(accuracy_val*100), verticalalignment='center', transform = ax.transAxes)
plt.rc('font',size=2)
plt.axis('off')
plt.show()
if i%10 == 0:
print(" "+str(i).zfill(5))
plt.rcParams['figure.figsize'] = (10.0, 5.0)
plt.rc('font',size=10)
plt.title("Accuacy over iterations")
plt.plot(train_accuracy_logs, c = 'r', label = 'training')
plt.plot(np.arange(0,iterations,batch_size), eval_accuracy_logs, c = 'black', label = 'testings')
plt.legend()
plt.show()
print(" completed {} iterations, during which {} epochs passed".format(iterations, epochs_completed))
return train_accuracy_logs, eval_accuracy_logs