So I wrote a TensorFlow CNN by creating manual layers. It is not state of art, but a simple experimental setup. The problem is it is not generalising well, it is hardly generalising. This should not be the case, it should at-least generalise somewhat.
As you will see below the loss hardly changes at all. I have considered adding drop out layer, but what is bugging me is the CNN does not generalise at all. What do you think is the problem here?
Here is my code:
learning_rate = 0.0001
epochs = 50
X = tf.placeholder(dtype = tf.float32, shape = (200, 32, 32, 3), name = 'Dataset_Placeholder')
Y = tf.placeholder(dtype = tf.float32, shape = (200, 2), name = 'Results_Placeholder')
X_CV = tf.placeholder(dtype = tf.float32, shape = (40, 32, 32, 3), name = 'CVDataset_Placeholder')
Y_CV = tf.placeholder(dtype = tf.float32, shape = (40, 2), name = 'CVResults_Placeholder')
weights_conv1 = tf.get_variable(name = 'wc1', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 3, 20), mean = 0, stddev = 0.1) )
weights_conv2 = tf.get_variable(name = 'wc2', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv3 = tf.get_variable(name = 'wc3', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv4 = tf.get_variable(name = 'wc4', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
weights_conv5 = tf.get_variable(name = 'wc5', dtype = tf.float32, initializer = tf.random_normal(shape = (3, 3, 20, 20), mean = 0, stddev = 0.1))
filters = [weights_conv1] + [weights_conv2] + [weights_conv3] + [weights_conv4] + [weights_conv5]
bias1 = tf.get_variable(name = 'b1', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias2 = tf.get_variable(name = 'b2', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias3 = tf.get_variable(name = 'b3', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias4 = tf.get_variable(name = 'b4', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
bias5 = tf.get_variable(name = 'b5', dtype = tf.float32, initializer = tf.random_normal(mean = 0, stddev = 0.001, shape = (1, 1, 1, 20)))
biases = [bias1] + [bias2] + [bias3] + [bias4] + [bias5]
def convolutionForwardPropagation(X):
c1 = tf.nn.conv2d(X, filters[0], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f1 = tf.nn.relu(c1 + biases[0])
c2 = tf.nn.conv2d(f1, filters[1], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f2 = tf.nn.relu(c2 + biases[1])
c3 = tf.nn.conv2d(f2, filters[2], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f3 = tf.nn.relu(c3 + biases[2])
c4 = tf.nn.conv2d(f3, filters[3], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f4 = tf.nn.relu(c4 + biases[3])
c5 = tf.nn.conv2d(f4, filters[4], strides =[1,1,1,1], data_format ='NHWC', padding = 'VALID')
f5 = tf.nn.leaky_relu(c5 + biases[4])
shape = f5.shape
fr = tf.reshape(f5,(shape[0], shape[3] * shape[2] * shape[1]))
fc1 = tf.contrib.layers.fully_connected(fr, 50, activation_fn = tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
fc2 = tf.contrib.layers.fully_connected(fc1, 2, activation_fn = tf.nn.sigmoid, weights_regularizer = tf.contrib.layers.l2_regularizer(5.0))
print(fc2.shape)
return fc2
fc2 = convolutionForwardPropagation(X)
entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits = fc2, labels = Y, name = 'cross_entropy')
loss = tf.reduce_mean(entropy, name = 'loss')
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
hypothesis = tf.nn.softmax(fc2)
y_pred_class = tf.argmax(hypothesis, axis = 1)
correct_preds = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_sum(tf.cast(correct_preds, tf.float32))
fcCV = convolutionForwardPropagation(X_CV)
entropyCV = tf.nn.softmax_cross_entropy_with_logits_v2(logits = fcCV, labels = Y_CV, name = 'cross_entropy')
lossCV = tf.reduce_mean(entropyCV, name = 'loss')
hypothesisCV = tf.nn.softmax(fcCV)
correct_predsCV = tf.equal(tf.argmax(hypothesisCV, 1), tf.argmax(Y_CV, 1))
accuracyCV = tf.reduce_sum(tf.cast(correct_predsCV, tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
l, h, c, ac = sess.run([lossCV, hypothesisCV, correct_predsCV, accuracyCV], feed_dict = {X_CV:CVDataset, Y_CV:y_CV})
print(ac, " ", l)
for i in range(0, epochs):
#sess.run(fc)
_, l, h, c, acc = sess.run([optimizer, loss, hypothesis, correct_preds, accuracy], feed_dict = {X:trainDataset, Y:y_train})
print("Epoch :", i+1, ", loss : ", l, ", accuracy :", acc)
l, h, c, ac = sess.run([lossCV, hypothesisCV, correct_predsCV, accuracyCV], feed_dict = {X_CV:CVDataset, Y_CV:y_CV})
print(ac, " ", l)
writer.close()
Here is my outputs (Removed dome of them so the post does not contain too much code):
20.0 0.70181745
Epoch : 1 , loss : 0.71289825 , accuracy : 100.0
Epoch : 3 , loss : 0.70839673 , accuracy : 100.0
Epoch : 5 , loss : 0.70380384 , accuracy : 100.0
Epoch : 7 , loss : 0.6992179 , accuracy : 100.0
Epoch : 9 , loss : 0.6949341 , accuracy : 103.0
Epoch : 11 , loss : 0.69119203 , accuracy : 111.0
Epoch : 13 , loss : 0.6879886 , accuracy : 111.0
Epoch : 15 , loss : 0.6850215 , accuracy : 110.0
Epoch : 17 , loss : 0.6818766 , accuracy : 113.0
Epoch : 18 , loss : 0.680143 , accuracy : 117.0
Epoch : 19 , loss : 0.6782758 , accuracy : 119.0
Epoch : 21 , loss : 0.6741557 , accuracy : 126.0
Epoch : 23 , loss : 0.66965437 , accuracy : 128.0
Epoch : 37 , loss : 0.635115 , accuracy : 140.0
Epoch : 39 , loss : 0.62959635 , accuracy : 139.0
Epoch : 41 , loss : 0.6239494 , accuracy : 145.0
Epoch : 43 , loss : 0.6180825 , accuracy : 147.0
Epoch : 45 , loss : 0.61196554 , accuracy : 153.0
Epoch : 47 , loss : 0.6056536 , accuracy : 154.0
Epoch : 49 , loss : 0.5992168 , accuracy : 156.0
Epoch : 50 , loss : 0.59595585 , accuracy : 155.0
20.0 0.7038449