# What's wrong with my deep NN of two hidden layers?

    batch_size = 128
size_1 = 1024
size_2 = 256
size_3 = 128
beta = 0.001

graph = tf.Graph()
with graph.as_default():

tf_train_dataset = tf.placeholder(
tf.float32,shape=(batch_size,image_size*image_size))
tf_train_labels = tf.placeholder(
tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)

# Weights and Biases
g_W1 = tf.Variable(
tf.truncated_normal([image_size*image_size,size_1]))
g_B1 = tf.Variable(
tf.zeros([size_1]))

g_W2 = tf.Variable(
tf.truncated_normal([size_1,size_2]))
g_B2 = tf.Variable(
tf.zeros([size_2]))

g_W3 = tf.Variable(
tf.truncated_normal([size_2,num_labels]))
g_B3 = tf.Variable(
tf.zeros([num_labels]))

#     g_W4 = tf.Variable(
#         tf.truncated_normal([size_3,num_labels]))
#     g_B4 = tf.Variable(
#         tf.zeros([num_labels]))

L1 = tf.nn.relu(
tf.matmul(tf_train_dataset,g_W1) + g_B1)
L2 = tf.nn.relu(
tf.matmul(L1,g_W2) + g_B2)
#     L3 = tf.nn.relu(
#         tf.matmul(L2,g_W3) + g_B3)

dr_prob = tf.placeholder("float")

#L1 = tf.nn.dropout(tf.nn.relu(
#   tf.matmul(tf_train_dataset,g_W1) + g_B1), 1.0)
#L2 = tf.nn.dropout(tf.nn.relu(
#   tf.matmul(L1,g_W2) + g_B2), 1.0)
#L3 = tf.nn.dropout(tf.nn.relu(
#   tf.matmul(L2,g_W3) + g_B3), 1.0)

logits = tf.matmul(L2, g_W3) + g_B3

loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))+\
beta*tf.nn.l2_loss(g_W1) +\
beta*tf.nn.l2_loss(g_W2)+\
beta*tf.nn.l2_loss(g_W3)
#         beta*tf.nn.l2_loss(g_W4)

# Optimizer.

# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
L1_pred = tf.nn.relu(tf.matmul(tf_valid_dataset, g_W1) + g_B1)
L2_pred = tf.nn.relu(tf.matmul(L1_pred, g_W2) + g_B2)
#     L3_pred = tf.nn.relu(tf.matmul(L2_pred, g_W3) + g_B3)
valid_prediction = tf.nn.softmax(tf.matmul(L2_pred, g_W3) + g_B3)

L1_test = tf.nn.relu(tf.matmul(tf_test_dataset, g_W1) + g_B1)
L2_test = tf.nn.relu(tf.matmul(L1_test, g_W2) + g_B2)
#     L3_test = tf.nn.relu(tf.matmul(L2_test, g_W3) + g_B3)
test_prediction = tf.nn.softmax(tf.matmul(L2_test, g_W3) + g_B3)

num_steps = 3001

with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
print("Initialized")
for step in range(num_steps):
# Pick an offset within the training data, which has been randomized.
# Note: we could use better randomization across epochs.
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
# Generate a minibatch.
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
# Prepare a dictionary telling the session where to feed the minibatch.
# The key of the dictionary is the placeholder node of the graph to be fed,
# and the value is the numpy array to feed to it.
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels, dr_prob : 0.5}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(
valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))


Now it's 2 days trying to know what's wrong with my solution, I hope somebody can spot it, the purpose is to train a simple deep NN of two hidden NN, I have checked other's solutions and I still don't get what's wrong with my code (it's 4th problem 3rd assignment of Udacity deep learning online course) I am getting the following output..

Initialized
Minibatch loss at step 0: 3983.812256
Minibatch accuracy: 8.6%
Validation accuracy: 10.0%
Minibatch loss at step 500: nan
Minibatch accuracy: 9.4%
Validation accuracy: 10.0%
Minibatch loss at step 1000: nan
Minibatch accuracy: 8.6%
Validation accuracy: 10.0%
Minibatch loss at step 1500: nan
Minibatch accuracy: 11.7%
Validation accuracy: 10.0%
Minibatch loss at step 2000: nan
Minibatch accuracy: 6.2%
Validation accuracy: 10.0%
Minibatch loss at step 2500: nan
Minibatch accuracy: 10.2%
Validation accuracy: 10.0%
Minibatch loss at step 3000: nan
Minibatch accuracy: 7.8%
Validation accuracy: 10.0%
Test accuracy: 10.0%


You didn't tell in your question what you tried when debugging, but I'll try to answer.

Short answer: It looks to me you got to choose a lower learning rate since your loss is exploding after the first iteration.

Explanation: You are using a standard Stochastic Gradient Descent to perform optimization. Therefore, it's an non-adaptive learning rate algorithms, which means if that latter is poorly chosen, loss can explode if the learning rate is too high. That's why when I'm running in such optimization issues with a new neural network, what I like to do is to set a very low learning rate to ensure convergence at first. Also you could use an adaptive optimizer such as AdaGrad or Adam who have both tensorflow implementations.

I hope that'll solve your issue.

• reducing learning rate to 0.05 solves the divergence problem, that's completely right, keeping the same code (not using adaptive learning rate) the only way to get a better accuracy now is to increase the number of steps, am I right? May 25 '18 at 16:41
• Well not necessarily, one problem. The only thing that's sure is that if your loss dimishes you can put away the optimization issue. If you plot the learning and validation loss you'll see if you have achieved convergence or not. If you have then increasing the number of steps is useless. Otherwise, you can increase the number of steps or refining the learning rate, or trying an adaptive optimizer. I hope I'm clear enough. May 25 '18 at 16:48

In addition to the reply I marked as the answer to my Question (the learning rate) I would like to add the following things that I needed to change:

1. The standard deviation since I initialized my weights as truncated normal,

2. Using a function that truncates the output of my Relu (relu6 in tensorflow).