1
$\begingroup$

I am a beginner in the Tensor Flow and I am trying to figure out the overfitting issue.

I took an example in a quite popular GitHub repo and modify a little bit.

https://github.com/floydhub/tensorflow-examples/blob/master/4_Utils/save_restore_model.py

Here is my question. According to every tutorial talking about overfitting issue, they said: too much training data will raise the accuracy on training data but lower the test data accuracy without proper regularization.

But this is what I plot from the following code. As you can see, the cost value indeed first lower and getting higher. However, the accuracy on both training data and test data raise first and decrease simultaneously. I am stuck here and don't know what to modify.

enter image description here

from __future__ import print_function
# Import MNIST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
import tensorflow as tf

# Parameters
learning_rate = 0.1
batch_size = 100
display_step = 1

# Network Parameters
n_hidden_1 = 30 # 1st layer
n_hidden_2 = 30 # 2nd layer
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
epochs = 30

# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])

cost_arr = []
test_ac_arr = []
train_ac_arr = []

# Create model
def multilayer_perceptron(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)
    # Hidden layer with RELU activation
    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    layer_2 = tf.nn.relu(layer_2)
    # Output layer with linear activation
    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
    return out_layer

# Store layers weight & bias
weights = {
    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
    'b2': tf.Variable(tf.random_normal([n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_classes]))
}

# Construct model
pred = multilayer_perceptron(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Initializing the variables
init = tf.global_variables_initializer()

# 'Saver' op to save and restore all the variables
saver = tf.train.Saver()

# Running first session
print("Starting 1st session...")
with tf.Session() as sess:
    # Initialize variables
    sess.run(init)

    # Training
    for epoch in range(epochs):
        avg_cost = 0.
        total_batch = int(mnist.train.num_examples/batch_size)

        # Feed with trainning data
        for i in range(total_batch):
            batch_x, batch_y = mnist.train.next_batch(batch_size)
            # Run optimization op (backprop) and cost op (to get loss value)
            _, c = sess.run([optimizer, cost], feed_dict={x: batch_x, y: batch_y})
            # Compute average loss
            avg_cost += c / total_batch

        # Display logs per epoch
        print("Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost))
        cost_arr.append(avg_cost)

        correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
        # Test accuracy
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
        test_ac = accuracy.eval({x: mnist.test.images, y: mnist.test.labels})
        print("Accuracy on Test Data Set:", test_ac)
        test_ac_arr.append(test_ac)

        # Training accuracy
        train_ac = accuracy.eval({x: mnist.train.images, y: mnist.train.labels})
        print("Accuracy on Training Data Set:", train_ac)
        train_ac_arr.append(train_ac)
        print("-------------------------------------")

Plot the data

import matplotlib.pyplot as plt

# Data for plotting
t = np.arange(0, len(cost_arr), 1)
s = cost_arr
# print(s)
fig, ax = plt.subplots()
ax.plot(t, s)
ax.set(xlabel='Epoch', ylabel='Cost Value',
       title='Cost Value on training data set')
ax.grid()
plt.show()


# Data for plotting AC on training Data Set
t = np.arange(0, len(cost_arr), 1)
s = test_ac_arr
# print(s)
fig, ax = plt.subplots()
ax.plot(t, s)
ax.set(xlabel='Epoch', ylabel='Test Accuracy',
       title='Accuracy on test data set')
ax.grid()
plt.show()


# Data for plotting AC on training Data Set
t = np.arange(0, len(cost_arr), 1)
s = test_ac_arr
# print(s)
fig, ax = plt.subplots()
ax.plot(t, s)
ax.set(xlabel='Epoch', ylabel='Training Accuracy',
       title='Accuracy on training data set')
ax.grid()
plt.show()
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.