0
$\begingroup$

So I wrote a TensorFlow CNN by creating manual layers. It is not state of art, but a simple experimental setup. The problem is it is not generalising well, it is hardly generalising. This should not be the case, it should at-least generalise somewhat.

As you will see below the loss hardly changes at all. I have considered adding drop out layer, but what is bugging me is the CNN does not generalise at all. What do you think is the problem here?

Here is my code:

learning_rate = 0.0001
epochs = 50
X = tf.placeholder(dtype = tf.float32, shape = (200, 32, 32, 3), name = 'Dataset_Placeholder')
Y = tf.placeholder(dtype = tf.float32, shape = (200, 2), name = 'Results_Placeholder')

X_CV = tf.placeholder(dtype = tf.float32, shape = (40, 32, 32, 3), name = 'CVDataset_Placeholder')
Y_CV = tf.placeholder(dtype = tf.float32, shape = (40, 2), name = 'CVResults_Placeholder')

weights_conv1 = tf.get_variable(name = 'wc1', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 3, 20), mean = 0, stddev = 0.1) )
weights_conv2 = tf.get_variable(name = 'wc2', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv3 = tf.get_variable(name = 'wc3', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv4 = tf.get_variable(name = 'wc4', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv5 = tf.get_variable(name = 'wc5', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
filters = [weights_conv1] + [weights_conv2] + [weights_conv3] + [weights_conv4] + [weights_conv5]

bias1 = tf.get_variable(name = 'b1', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias2 = tf.get_variable(name = 'b2', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias3 = tf.get_variable(name = 'b3', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias4 = tf.get_variable(name = 'b4', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias5 = tf.get_variable(name = 'b5', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
biases = [bias1] + [bias2] + [bias3] + [bias4] + [bias5]


def convolutionForwardPropagation(X):
    c1 = tf.nn.conv2d(X, filters[0], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
    f1 = tf.nn.relu(c1 + biases[0])
    c2 = tf.nn.conv2d(f1, filters[1], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
    f2 = tf.nn.relu(c2 + biases[1])

    c3 = tf.nn.conv2d(f2, filters[2], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
    f3 = tf.nn.relu(c3 + biases[2])
    c4 = tf.nn.conv2d(f3, filters[3], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
    f4 = tf.nn.relu(c4 + biases[3])

    c5 = tf.nn.conv2d(f4, filters[4], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
    f5 = tf.nn.leaky_relu(c5 + biases[4])

    shape = f5.shape
    fr = tf.reshape(f5,(shape[0], shape[3] * shape[2] * shape[1]))
    fc1 = tf.contrib.layers.fully_connected(fr, 50, activation_fn = tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
    fc2 = tf.contrib.layers.fully_connected(fc1, 2, activation_fn = tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
    print(fc2.shape)
    return fc2


fc2 = convolutionForwardPropagation(X)
entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits = fc2, labels = Y, name = 'cross_entropy')
loss = tf.reduce_mean(entropy, name = 'loss')

optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)

hypothesis = tf.nn.softmax(fc2)
y_pred_class = tf.argmax(hypothesis, axis = 1)
correct_preds = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_sum(tf.cast(correct_preds, tf.float32))

fcCV = convolutionForwardPropagation(X_CV)
entropyCV = tf.nn.softmax_cross_entropy_with_logits_v2(logits = fcCV, labels = Y_CV, name = 'cross_entropy')
lossCV = tf.reduce_mean(entropyCV, name = 'loss')

hypothesisCV = tf.nn.softmax(fcCV)
correct_predsCV = tf.equal(tf.argmax(hypothesisCV, 1), tf.argmax(Y_CV, 1))
accuracyCV = tf.reduce_sum(tf.cast(correct_predsCV, tf.float32))

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())        
    l, h, c, ac = sess.run([lossCV, hypothesisCV, correct_predsCV, accuracyCV], feed_dict = {X_CV:CVDataset, Y_CV:y_CV})
    print(ac, " ", l)
    for i in range(0, epochs):
        #sess.run(fc)
        _, l, h, c, acc = sess.run([optimizer, loss, hypothesis, correct_preds, accuracy], feed_dict = {X:trainDataset, Y:y_train})                 
        print("Epoch :", i+1, ", loss : ", l, ", accuracy :", acc)
    l, h, c, ac = sess.run([lossCV, hypothesisCV, correct_predsCV, accuracyCV], feed_dict = {X_CV:CVDataset, Y_CV:y_CV})
    print(ac, " ", l)
writer.close()

Here is my outputs (Removed dome of them so the post does not contain too much code):

20.0   0.70181745
Epoch : 1 , loss :  0.71289825 , accuracy : 100.0
Epoch : 3 , loss :  0.70839673 , accuracy : 100.0
Epoch : 5 , loss :  0.70380384 , accuracy : 100.0
Epoch : 7 , loss :  0.6992179 , accuracy : 100.0
Epoch : 9 , loss :  0.6949341 , accuracy : 103.0
Epoch : 11 , loss :  0.69119203 , accuracy : 111.0
Epoch : 13 , loss :  0.6879886 , accuracy : 111.0
Epoch : 15 , loss :  0.6850215 , accuracy : 110.0
Epoch : 17 , loss :  0.6818766 , accuracy : 113.0
Epoch : 18 , loss :  0.680143 , accuracy : 117.0
Epoch : 19 , loss :  0.6782758 , accuracy : 119.0
Epoch : 21 , loss :  0.6741557 , accuracy : 126.0
Epoch : 23 , loss :  0.66965437 , accuracy : 128.0
Epoch : 37 , loss :  0.635115 , accuracy : 140.0
Epoch : 39 , loss :  0.62959635 , accuracy : 139.0
Epoch : 41 , loss :  0.6239494 , accuracy : 145.0
Epoch : 43 , loss :  0.6180825 , accuracy : 147.0
Epoch : 45 , loss :  0.61196554 , accuracy : 153.0
Epoch : 47 , loss :  0.6056536 , accuracy : 154.0
Epoch : 49 , loss :  0.5992168 , accuracy : 156.0
Epoch : 50 , loss :  0.59595585 , accuracy : 155.0
20.0   0.7038449
Improve this question
$\endgroup$

2 Answers 2

Reset to default
1
$\begingroup$

When you do fcCV = convolutionForwardPropagation(X_CV) you create a separate graph than the one you train on, so the CV graph is never updated.

To fix this you can change the X and y placeholders to have a variable batch size by specifying a None for the batch dimension:

X = tf.placeholder(dtype = tf.float32, shape = (None, 32, 32, 3), name = 'Dataset_Placeholder')
Y = tf.placeholder(dtype = tf.float32, shape = (None, 2), name = 'Results_Placeholder')

Then when you run your validation set, just use those X and y placeholders.

EDIT:

I realized you are actually reusing the weights you defined at the top for the convolution layers, but the fully connected layers are defined using tf.contrib.layers.fully_connected which will create new weights the second time it is called, so the above should still fix it. Another way to fix this is to define the weights for the fully connected layer as you did with the convolution layers and use a tf.matmul, but using the same placeholder for train and val input is probably cleaner.

Improve this answer
$\endgroup$
4
  • $\begingroup$ Thanks man my instinct was telling me the weights definitely re not the same....still let me check it out.... $\endgroup$
    – DuttaA
    Jul 23, 2018 at 3:30
  • $\begingroup$ Nopes, i also thought tf.contrib.layers.fully_connected was the culprit...its not, i checked it...by any chance do you know how to set the dropout to 1 when i am doing the CV set $\endgroup$
    – DuttaA
    Jul 23, 2018 at 3:43
  • $\begingroup$ That will definitely create two sets of fully connected weights if you call convolutionForwardPropagation twice. You can check how many fully connected weight variables got created by calling tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='.*fully_connected.*'). There should only be 2 since there are 2 fully_connected layers in your graph but I suspect there are 4. $\endgroup$
    – kenny
    Jul 23, 2018 at 4:05
  • $\begingroup$ But i called the function after evaluating CV with the train set i.e. after evaluating CV, training, CV, then again train set...turns out it gave almost same result as the accuracy where it left off while training...so I am guessing it creates new weights for new datasets....anyways your approach of converting it to a single function call worked $\endgroup$
    – DuttaA
    Jul 23, 2018 at 4:08
1
$\begingroup$

Try adding some max pooling layers after your convolutional layers. They will aggregate portions of your conv layers, which in turn will generalize better. Also, you could try adding drop out layers at the end of each convolutional block which deactivate a set percentage of nodes which in turn force other nodes to work harder thus generalizing better. Here is a quick edit of the general idea. When predicting your test set, you will want to set the dropout value to False in the placeholder_with_default if you decide to use it. Repeat this recipe for each layer.

learning_rate = 0.0001
epochs = 50
X = tf.placeholder(dtype = tf.float32, shape = (200, 32, 32, 3), name = 
'Dataset_Placeholder')
Y = tf.placeholder(dtype = tf.float32, shape = (200, 2), name = 
'Results_Placeholder')

batch_norm1 = tf.placeholder_with_default(True, shape = (None))
drop_holder  = tf.placeholder_with_default(True, shape = (None))

X_CV = tf.placeholder(dtype = tf.float32, shape = (40, 32, 32, 3), name = 
'CVDataset_Placeholder')
Y_CV = tf.placeholder(dtype = tf.float32, shape = (40, 2), name = 
'CVResults_Placeholder')

weights_conv1 = tf.get_variable(name = 'wc1', dtype = tf.float32, 
initializer = tf.random_normal(shape = (3, 3, 3, 20), mean = 0, stddev = 
0.1) )
weights_conv2 = tf.get_variable(name = 'wc2', dtype = tf.float32, 
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 
0.1))
weights_conv3 = tf.get_variable(name = 'wc3', dtype = tf.float32, 
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 
0.1))
weights_conv4 = tf.get_variable(name = 'wc4', dtype = tf.float32, 
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 
0.1))
weights_conv5 = tf.get_variable(name = 'wc5', dtype = tf.float32, 
initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 
0.1))
filters = [weights_conv1] + [weights_conv2] + [weights_conv3] + 
[weights_conv4] + [weights_conv5]

bias1 = tf.get_variable(name = 'b1', dtype = tf.float32, initializer = 
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias2 = tf.get_variable(name = 'b2', dtype = tf.float32, initializer = 
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias3 = tf.get_variable(name = 'b3', dtype = tf.float32, initializer = 
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias4 = tf.get_variable(name = 'b4', dtype = tf.float32, initializer = 
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias5 = tf.get_variable(name = 'b5', dtype = tf.float32, initializer = 
tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
biases = [bias1] + [bias2] + [bias3] + [bias4] + [bias5]


def convolutionForwardPropagation(X):
    c1 = tf.nn.conv2d(X, filters[0], strides =[1,1,1,1], data_format 
     ='NHWC', padding = 'VALID')
max_pool1 = tf.layers.max_pooling2d(c1, [2,2], strides = [2,2])
f1 = tf.nn.relu(max_pool1 + biases[0])
bn1 = tf.layers.batch_normalization(f1, training = batch_norm1)
#drop1 = tf.layers.dropout(bn1, 0.3, training = drop_holder)
c2 = tf.nn.conv2d(bn1, filters[1], strides =[1,1,1,1], data_format ='NHWC', 
padding = 'VALID')
f2 = tf.nn.relu(c2 + biases[1])

c3 = tf.nn.conv2d(f2, filters[2], strides =[1,1,1,1], data_format ='NHWC', 
padding = 'VALID')
f3 = tf.nn.relu(c3 + biases[2])
c4 = tf.nn.conv2d(f3, filters[3], strides =[1,1,1,1], data_format ='NHWC', 
padding = 'VALID')
f4 = tf.nn.relu(c4 + biases[3])

c5 = tf.nn.conv2d(f4, filters[4], strides =[1,1,1,1], data_format ='NHWC', 
padding = 'VALID')
f5 = tf.nn.leaky_relu(c5 + biases[4])

shape = f5.shape
fr = tf.reshape(f5,(shape[0], shape[3] * shape[2] * shape[1]))
fc1 = tf.contrib.layers.fully_connected(fr, 50, activation_fn = 
tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
fc2 = tf.contrib.layers.fully_connected(fc1, 2, activation_fn = 
tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
print(fc2.shape)
return fc2
Improve this answer
$\endgroup$
4
  • $\begingroup$ Pooling and dropout would not explain why the model does not do better than random. $\endgroup$
    – kenny
    Jul 23, 2018 at 3:41
  • $\begingroup$ How do you set dropout to false? Since as per the above solution i will be using a single placeholder which has no way of determining whether the data passed is CV or train $\endgroup$
    – DuttaA
    Jul 23, 2018 at 3:57
  • $\begingroup$ I typically only set to false for actual test predictions though I'd imagine you could try something like if epoch % 5 == 0:drop_holder = tf.placeholder_with_default(False, shape = (None)) then evaluate your cv $\endgroup$
    – stephen barter
    Jul 23, 2018 at 3:59
  • $\begingroup$ I tried it and it works. It's not the prettiest but it does the job. If you wanted to keep it clean you could put your training block in a function with a argument for training (True or False). $\endgroup$
    – stephen barter
    Jul 23, 2018 at 22:18

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.