Why is this TensorFlow CNN not generalising?

So I wrote a TensorFlow CNN by creating manual layers. It is not state of art, but a simple experimental setup. The problem is it is not generalising well, it is hardly generalising. This should not be the case, it should at-least generalise somewhat.

As you will see below the loss hardly changes at all. I have considered adding drop out layer, but what is bugging me is the CNN does not generalise at all. What do you think is the problem here?

Here is my code:

learning_rate = 0.0001
epochs = 50
X = tf.placeholder(dtype = tf.float32, shape = (200, 32, 32, 3), name = 'Dataset_Placeholder')
Y = tf.placeholder(dtype = tf.float32, shape = (200, 2), name = 'Results_Placeholder')

X_CV = tf.placeholder(dtype = tf.float32, shape = (40, 32, 32, 3), name = 'CVDataset_Placeholder')
Y_CV = tf.placeholder(dtype = tf.float32, shape = (40, 2), name = 'CVResults_Placeholder')

weights_conv1 = tf.get_variable(name = 'wc1', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 3, 20), mean = 0, stddev = 0.1) )
weights_conv2 = tf.get_variable(name = 'wc2', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv3 = tf.get_variable(name = 'wc3', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv4 = tf.get_variable(name = 'wc4', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv5 = tf.get_variable(name = 'wc5', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
filters = [weights_conv1] + [weights_conv2] + [weights_conv3] + [weights_conv4] + [weights_conv5]

bias1 = tf.get_variable(name = 'b1', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias2 = tf.get_variable(name = 'b2', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias3 = tf.get_variable(name = 'b3', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias4 = tf.get_variable(name = 'b4', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias5 = tf.get_variable(name = 'b5', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
biases = [bias1] + [bias2] + [bias3] + [bias4] + [bias5]

def convolutionForwardPropagation(X):
c1 = tf.nn.conv2d(X, filters[0], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f1 = tf.nn.relu(c1 + biases[0])
c2 = tf.nn.conv2d(f1, filters[1], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f2 = tf.nn.relu(c2 + biases[1])

c3 = tf.nn.conv2d(f2, filters[2], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f3 = tf.nn.relu(c3 + biases[2])
c4 = tf.nn.conv2d(f3, filters[3], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f4 = tf.nn.relu(c4 + biases[3])

c5 = tf.nn.conv2d(f4, filters[4], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f5 = tf.nn.leaky_relu(c5 + biases[4])

shape = f5.shape
fr = tf.reshape(f5,(shape[0], shape[3] * shape[2] * shape[1]))
fc1 = tf.contrib.layers.fully_connected(fr, 50, activation_fn = tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
fc2 = tf.contrib.layers.fully_connected(fc1, 2, activation_fn = tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
print(fc2.shape)
return fc2

fc2 = convolutionForwardPropagation(X)
entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits = fc2, labels = Y, name = 'cross_entropy')
loss = tf.reduce_mean(entropy, name = 'loss')

hypothesis = tf.nn.softmax(fc2)
y_pred_class = tf.argmax(hypothesis, axis = 1)
correct_preds = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_sum(tf.cast(correct_preds, tf.float32))

fcCV = convolutionForwardPropagation(X_CV)
entropyCV = tf.nn.softmax_cross_entropy_with_logits_v2(logits = fcCV, labels = Y_CV, name = 'cross_entropy')
lossCV = tf.reduce_mean(entropyCV, name = 'loss')

hypothesisCV = tf.nn.softmax(fcCV)
correct_predsCV = tf.equal(tf.argmax(hypothesisCV, 1), tf.argmax(Y_CV, 1))
accuracyCV = tf.reduce_sum(tf.cast(correct_predsCV, tf.float32))

with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
l, h, c, ac = sess.run([lossCV, hypothesisCV, correct_predsCV, accuracyCV], feed_dict = {X_CV:CVDataset, Y_CV:y_CV})
print(ac, " ", l)
for i in range(0, epochs):
#sess.run(fc)
_, l, h, c, acc = sess.run([optimizer, loss, hypothesis, correct_preds, accuracy], feed_dict = {X:trainDataset, Y:y_train})
print("Epoch :", i+1, ", loss : ", l, ", accuracy :", acc)
l, h, c, ac = sess.run([lossCV, hypothesisCV, correct_predsCV, accuracyCV], feed_dict = {X_CV:CVDataset, Y_CV:y_CV})
print(ac, " ", l)
writer.close()


Here is my outputs (Removed dome of them so the post does not contain too much code):

20.0   0.70181745
Epoch : 1 , loss :  0.71289825 , accuracy : 100.0
Epoch : 3 , loss :  0.70839673 , accuracy : 100.0
Epoch : 5 , loss :  0.70380384 , accuracy : 100.0
Epoch : 7 , loss :  0.6992179 , accuracy : 100.0
Epoch : 9 , loss :  0.6949341 , accuracy : 103.0
Epoch : 11 , loss :  0.69119203 , accuracy : 111.0
Epoch : 13 , loss :  0.6879886 , accuracy : 111.0
Epoch : 15 , loss :  0.6850215 , accuracy : 110.0
Epoch : 17 , loss :  0.6818766 , accuracy : 113.0
Epoch : 18 , loss :  0.680143 , accuracy : 117.0
Epoch : 19 , loss :  0.6782758 , accuracy : 119.0
Epoch : 21 , loss :  0.6741557 , accuracy : 126.0
Epoch : 23 , loss :  0.66965437 , accuracy : 128.0
Epoch : 37 , loss :  0.635115 , accuracy : 140.0
Epoch : 39 , loss :  0.62959635 , accuracy : 139.0
Epoch : 41 , loss :  0.6239494 , accuracy : 145.0
Epoch : 43 , loss :  0.6180825 , accuracy : 147.0
Epoch : 45 , loss :  0.61196554 , accuracy : 153.0
Epoch : 47 , loss :  0.6056536 , accuracy : 154.0
Epoch : 49 , loss :  0.5992168 , accuracy : 156.0
Epoch : 50 , loss :  0.59595585 , accuracy : 155.0
20.0   0.7038449


When you do fcCV = convolutionForwardPropagation(X_CV) you create a separate graph than the one you train on, so the CV graph is never updated.

To fix this you can change the X and y placeholders to have a variable batch size by specifying a None for the batch dimension:

X = tf.placeholder(dtype = tf.float32, shape = (None, 32, 32, 3), name = 'Dataset_Placeholder')
Y = tf.placeholder(dtype = tf.float32, shape = (None, 2), name = 'Results_Placeholder')


Then when you run your validation set, just use those X and y placeholders.

EDIT:

I realized you are actually reusing the weights you defined at the top for the convolution layers, but the fully connected layers are defined using tf.contrib.layers.fully_connected which will create new weights the second time it is called, so the above should still fix it. Another way to fix this is to define the weights for the fully connected layer as you did with the convolution layers and use a tf.matmul, but using the same placeholder for train and val input is probably cleaner.

• Thanks man my instinct was telling me the weights definitely re not the same....still let me check it out.... – DuttaA Jul 23 '18 at 3:30
• Nopes, i also thought tf.contrib.layers.fully_connected was the culprit...its not, i checked it...by any chance do you know how to set the dropout to 1 when i am doing the CV set – DuttaA Jul 23 '18 at 3:43
• That will definitely create two sets of fully connected weights if you call convolutionForwardPropagation twice. You can check how many fully connected weight variables got created by calling tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='.*fully_connected.*'). There should only be 2 since there are 2 fully_connected layers in your graph but I suspect there are 4. – kenny Jul 23 '18 at 4:05
• But i called the function after evaluating CV with the train set i.e. after evaluating CV, training, CV, then again train set...turns out it gave almost same result as the accuracy where it left off while training...so I am guessing it creates new weights for new datasets....anyways your approach of converting it to a single function call worked – DuttaA Jul 23 '18 at 4:08

Try adding some max pooling layers after your convolutional layers. They will aggregate portions of your conv layers, which in turn will generalize better. Also, you could try adding drop out layers at the end of each convolutional block which deactivate a set percentage of nodes which in turn force other nodes to work harder thus generalizing better. Here is a quick edit of the general idea. When predicting your test set, you will want to set the dropout value to False in the placeholder_with_default if you decide to use it. Repeat this recipe for each layer.

learning_rate = 0.0001
epochs = 50
X = tf.placeholder(dtype = tf.float32, shape = (200, 32, 32, 3), name =
'Dataset_Placeholder')
Y = tf.placeholder(dtype = tf.float32, shape = (200, 2), name =
'Results_Placeholder')

batch_norm1 = tf.placeholder_with_default(True, shape = (None))
drop_holder  = tf.placeholder_with_default(True, shape = (None))

X_CV = tf.placeholder(dtype = tf.float32, shape = (40, 32, 32, 3), name =
'CVDataset_Placeholder')
Y_CV = tf.placeholder(dtype = tf.float32, shape = (40, 2), name =
'CVResults_Placeholder')

weights_conv1 = tf.get_variable(name = 'wc1', dtype = tf.float32,
initializer = tf.random_normal(shape = (3, 3, 3, 20), mean = 0, stddev =
0.1) )
weights_conv2 = tf.get_variable(name = 'wc2', dtype = tf.float32,
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev =
0.1))
weights_conv3 = tf.get_variable(name = 'wc3', dtype = tf.float32,
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev =
0.1))
weights_conv4 = tf.get_variable(name = 'wc4', dtype = tf.float32,
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev =
0.1))
weights_conv5 = tf.get_variable(name = 'wc5', dtype = tf.float32,
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev =
0.1))
filters = [weights_conv1] + [weights_conv2] + [weights_conv3] +
[weights_conv4] + [weights_conv5]

bias1 = tf.get_variable(name = 'b1', dtype = tf.float32, initializer =
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias2 = tf.get_variable(name = 'b2', dtype = tf.float32, initializer =
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias3 = tf.get_variable(name = 'b3', dtype = tf.float32, initializer =
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias4 = tf.get_variable(name = 'b4', dtype = tf.float32, initializer =
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias5 = tf.get_variable(name = 'b5', dtype = tf.float32, initializer =
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
biases = [bias1] + [bias2] + [bias3] + [bias4] + [bias5]

def convolutionForwardPropagation(X):
c1 = tf.nn.conv2d(X, filters[0], strides =[1,1,1,1], data_format
max_pool1 = tf.layers.max_pooling2d(c1, [2,2], strides = [2,2])
f1 = tf.nn.relu(max_pool1 + biases[0])
bn1 = tf.layers.batch_normalization(f1, training = batch_norm1)
#drop1 = tf.layers.dropout(bn1, 0.3, training = drop_holder)
c2 = tf.nn.conv2d(bn1, filters[1], strides =[1,1,1,1], data_format ='NHWC',
f2 = tf.nn.relu(c2 + biases[1])

c3 = tf.nn.conv2d(f2, filters[2], strides =[1,1,1,1], data_format ='NHWC',
f3 = tf.nn.relu(c3 + biases[2])
c4 = tf.nn.conv2d(f3, filters[3], strides =[1,1,1,1], data_format ='NHWC',
f4 = tf.nn.relu(c4 + biases[3])

c5 = tf.nn.conv2d(f4, filters[4], strides =[1,1,1,1], data_format ='NHWC',
f5 = tf.nn.leaky_relu(c5 + biases[4])

shape = f5.shape
fr = tf.reshape(f5,(shape[0], shape[3] * shape[2] * shape[1]))
fc1 = tf.contrib.layers.fully_connected(fr, 50, activation_fn =
tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
fc2 = tf.contrib.layers.fully_connected(fc1, 2, activation_fn =
tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
print(fc2.shape)
return fc2

• Pooling and dropout would not explain why the model does not do better than random. – kenny Jul 23 '18 at 3:41
• How do you set dropout to false? Since as per the above solution i will be using a single placeholder which has no way of determining whether the data passed is CV or train – DuttaA Jul 23 '18 at 3:57
• I typically only set to false for actual test predictions though I'd imagine you could try something like if epoch % 5 == 0:drop_holder = tf.placeholder_with_default(False, shape = (None)) then evaluate your cv – stephen barter Jul 23 '18 at 3:59
• I tried it and it works. It's not the prettiest but it does the job. If you wanted to keep it clean you could put your training block in a function with a argument for training (True or False). – stephen barter Jul 23 '18 at 22:18