2
$\begingroup$

I have made sure that layers, parameters, hyperparameters, kernel_initialization, bias_initialization, seeds and datasets are all equal. But still the output for both the models are different.

    from __future__ import division, print_function, absolute_import
import tensorflow.keras.datasets.mnist as mnist
import tensorflow as tf
tf.reset_default_graph()
tf.random.set_random_seed(43)
from input_data import Dataset
# Import MNIST data
import input_data
# mnist = input_data_old.read_data_sets("MNSITData/raw/", one_hot=True)

# Seed value
# Apparently you may use different seed values at each stage
seed_value= 43

# 1. Set the `PYTHONHASHSEED` environment variable at a fixed value
import os
os.environ['PYTHONHASHSEED']=str(seed_value)

# 2. Set the `python` built-in pseudo-random generator at a fixed value
import random
random.seed(seed_value)

# 3. Set the `numpy` pseudo-random generator at a fixed value
import numpy as np
np.random.seed(seed_value)

# 4. Set the `tensorflow` pseudo-random generator at a fixed value
tf.set_random_seed(seed_value)
# (x_train, y_train), (x_test, y_test) = mnist.load_data()
# x_train, x_test = x_train / 255, x_test / 255

# # Add a channels dimension
# x_train = x_train[..., None]
# x_test = x_test[..., None]
# train_ds = tf.data.Dataset.from_tensor_slices(
#     (x_train, y_train)).batch(64)
# test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(64)
# Training Parameters
learning_rate = 0.001
batch = 5
display_step = 10

dataset={
    "file_path":"/home/pat-011/Desktop/deep_learning/playground/flower_photos",
    "ratio":(0.8,0.1,0.1),
    "target_size":(150,150),
    "color_mode":"rgb",
    "class_mode":'sparse'
}
train,test,val_ds= Dataset(dataset)()
np.save("train",train[0])
np.save("test",test[0])

# Network Parameters
num_classes = 5# MNIST total classes (0-9 digits)
dropout = 0.25 # Dropout, probability to keep units

# tf Graph input
X = tf.placeholder(tf.float32, [None, 150,150,3])
Y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32) # dropout (keep probability)


# Create some wrappers for simplicity
def conv2d(x, W, b, strides=1):
    print("shape of x",x.shape.as_list(),type(x.shape.as_list()),x.shape.as_list()[0],x.shape.as_list()[1])
    # Conv2D wrapper, with bias and relu activation
    tf.compat.v1.summary.scalar('conv_weights', W )
    tf.compat.v1.summary.scalar('conv_bias', b)
    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='VALID')
    x = tf.nn.bias_add(x, b)
    return tf.nn.relu(x)

def maxpool2d(x, k=2):
    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],
                          padding='VALID')


# Create model
def conv_net(x, weights, biases):
    # MNIST data input is a 1-D vector of 784 features (128*128 pixels)
    # Reshape to match picture format [Height x Width x Channel]
    # Tensor input become 4-D: [Batch Size, Height, Width, Channel]
    print(type(x.shape[-1]),x.shape[-1],type(int(x.shape[-1])))
    x = tf.reshape(x, shape=[-1, 150, 150, 3])
    # Convolution Layerint()
    conv1 = conv2d(x, weights['wc1'], biases['bc1'])
    tf.compat.v1.summary.scalar(conv1.name, conv1)
    # Max Pooling (down-sampling)
    print(conv1.shape)
    conv1 = maxpool2d(conv1, k=2)
    print(conv1.shape)
    # Convolution Layer
    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])
    # Max Pooling (down-sampling)
    print(conv2.shape)
    conv2 = maxpool2d(conv2, k=2)
    print(conv2.shape)

    # Fully connected layer
    # Reshape conv2 output to fit fully connected layer input
    print("weights_____________________________",[-1, weights['wd1'].get_shape().as_list()[0]])

    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])
    print(fc1.shape,)
    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])
    fc1 = tf.nn.relu(fc1)
    # Apply Dropout
    print("dropout++++++++++++++++++++++++++++++++++",dropout)
    fc1 = tf.nn.dropout(fc1, rate = dropout,seed=43)
    print(fc1.shape)
    
    fc1 = tf.reshape(fc1, [-1, weights['out'].get_shape().as_list()[0]])
    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
    tf.compat.v1.summary.scalar('dense_weights', weights['out'] )
    tf.compat.v1.summary.scalar('dense_bias', biases['out'])
    print(out.shape)
    tf.compat.v1.summary.scalar(out.name, out)
    return out

# Store layers weight & bias
weights = {
    # 5x5 conv, 1 input, 32 outputs
    'wc1': tf.Variable(tf.random_normal([5, 5, 3, 32],mean=0.0,
    stddev=1.0, seed=43)),
    # 'wc1': tf.Variable(np.load("conv_w.npy")),
    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64],mean=0.0,
    stddev=1.0, seed=43)),
    # fully connected, 7*7*64 inputs, 1024 outputs
    'wd1': tf.Variable(tf.random_normal([34*34*64, 1024],mean=0.0,
    stddev=1.0, seed=43)),
    # 1024 inputs, 10 outputs (class prediction)
    # 'out': tf.Variable(np.load("dense_w.npy"))
    'out': tf.Variable(tf.random_normal([1024, num_classes],mean=0.0,
    stddev=1.0, seed=43))
}

biases = {
    'bc1': tf.Variable(tf.random_normal([32],mean=0.0,
    stddev=1.0, seed=43)),
    # 'bc1': tf.Variable(np.load("conv_b.npy")),
    'bc2': tf.Variable(tf.random_normal([64],mean=0.0,
    stddev=1.0, seed=43)),
    'bd1': tf.Variable(tf.random_normal([1024],mean=0.0,
    stddev=1.0, seed=43)),
    'out': tf.Variable(tf.random_normal([num_classes], mean=0.0,
    stddev=1.0, seed=43))
    # 'out': tf.Variable(np.load("dense_b.npy"))
}
print(weights,biases)
# Construct model
logits = conv_net(X, weights, biases)
prediction = tf.nn.softmax(logits)

# Define loss and optimizer
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
    logits=logits, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)
tf.compat.v1.summary.scalar('loss_op', loss_op)

# Evaluate model
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.compat.v1.summary.scalar('accuracy', accuracy)

# Merge all the summaries and write them out to
# /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default)
merged = tf.compat.v1.summary.merge_all()
# Initialize thcorrect_prede variables (i.e. assign their default value)
init = tf.global_variables_initializer()

# Start training
with tf.Session() as sess:
    writer = tf.summary.FileWriter("/tmp/tf_withoutjson/",sess.graph)
    # Run the initializer
    sess.run(init)
    for i in range(5):
        for step in range(1, int(len(train[0])/batch)):
            # Run optimization op (backprop)
            offset = (step * batch) % (train[1].shape[0] - batch)
            # Generate a minibatch.
            batch_x = train[0][offset:(offset + batch), :]
            batch_y = train[1][offset:(offset + batch), :]
            # np.save("arrays/xtrain"+str(step),batch_x)
            # np.save("arrays/ytrain"+str(step),batch_y)
            sess.run(train_op, feed_dict={X: batch_x, Y: batch_y})
            if step % display_step == 0 or step == 1:
                # Calculate batch loss and accuracy
               
                loss, acc ,cw,cb,dw,db= sess.run([loss_op, accuracy,weights["wc1"],biases["bc1"],weights["out"],biases['out']], feed_dict={X: batch_x,
                                                                    Y: batch_y
                                                                    })
                np.save("weightBias/conv_w"+str(step),cw)
                np.save("weightBias/conv_b"+str(step),cb)
                np.save("weightBias/dense_w"+str(step),dw)
                np.save("weightBias/dense_b"+str(step),db)
                print("Step " + str(step) + ", Minibatch Loss= " + \
                    "{:.4f}".format(loss) + ", Training Accuracy= " + \
                    "{:.3f}".format(acc))

    print("Optimization Finished!")

    # Calculate accuracy for 256 MNIST test images
    print("Testing Accuracy:", \
        sess.run(accuracy, feed_dict={X: test[0],
                                      Y: test[1],
                                      keep_prob: 0.75}))

This is the TensorFlow flow and I have also saved the weights generated in each layer.

Now for the Keras flow,

import keras
from keras.layers import *
from keras.models import Model
from keras import optimizers
from keras.datasets import mnist
from keras import backend as K
from keras.utils import multi_gpu_model
import tensorflow as tf
from input_data import Dataset
from numpy import sqrt
from keras import initializers

# Seed value
# Apparently you may use different seed values at each stage
seed_value= 43

# 1. Set the `PYTHONHASHSEED` environment variable at a fixed value
import os
os.environ['PYTHONHASHSEED']=str(seed_value)

# 2. Set the `python` built-in pseudo-random generator at a fixed value
import random
random.seed(seed_value)

# 3. Set the `numpy` pseudo-random generator at a fixed value
import numpy as np
np.random.seed(seed_value)

# 4. Set the `tensorflow` pseudo-random generator at a fixed value
import tensorflow as tf
tf.set_random_seed(seed_value)

# 5. Configure a new global `tensorflow` session
from keras import backend as K
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)


img_rows, img_cols = 150, 150

dataset={
    "file_path":"/home/pat-011/Desktop/deep_learning/playground/flower_photos",
    "ratio":(0.8,0.1,0.1),
    "target_size":(150,150),
    "color_mode":"rgb",
    "class_mode":'sparse'
}

(x_train, y_train), (x_test, y_test), (x_val, y_val) = Dataset(dataset)()

in_shape = x_train.shape[1]
out_shape = y_train.shape[1]
n_train = x_train.shape[0]
n_test = x_test.shape[0]


# def get_network_utils():
#     dropout_ph = tf.placeholder_with_default(0.25, shape=())
#     optimizer = tf.train.AdamOptimizer(learning_rate=lr, name='Trainer')
#     activation = tf.nn.relu

#     def loss(y_true, y_pred):
#         return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred))

#     def accuracy(y_true, y_pred):
#         correct = tf.equal(tf.argmax(y_true, 1), tf.argmax(y_pred, 1))
#         return tf.reduce_mean(tf.cast(correct, tf.float32))

#     def weight_initializer(shape, dtype=None, partition_info=None):
#         init_range = sqrt(6.0 / (shape[0] + shape[1]))
#         return tf.get_variable('weights', shape=shape, dtype=dtype,
#                                initializer=tf.random_uniform_initializer(-init_range, init_range))

#     def bias_initializer(shape, dtype=None, partition_info=None):
#         return tf.Variable(name='bias', initial_value=tf.random_normal(shape))

#     return (dropout_ph, optimizer, activation, loss, accuracy,
#             weight_initializer, bias_initializer)

x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

lr = 0.001

# _, optimizer, activation, loss, accuracy, weight_initializer, bias_initializer = get_network_utils()

# # convert class vectors to binary class matrices
# y_train = keras.utils.to_categorical(y_train, num_classes)
# y_test = keras.utils.to_categorical(y_test, num_classes)

inputs = Input(shape=(150, 150, 3))
x = Conv2D(32, (5, 5), activation='relu', strides = (1, 1), padding = 'VALID', kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0,seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(inputs)
x = MaxPool2D(pool_size=2, strides = (2, 2), padding = 'VALID')(x)
x = Conv2D(64, (5, 5),activation='relu', strides = (1, 1), padding = 'VALID',kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(x)
x = MaxPool2D(pool_size=2, strides = (2, 2), padding = 'VALID')(x)
x = Flatten()(x)
x = Dense(1024, activation='relu', kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(x)
x = Dropout(rate = 0.25, seed = 43)(x)
predictions = Dense(5, activation='softmax', kernel_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43), bias_initializer=initializers.RandomNormal(mean=0.0,
    stddev=1.0, seed = 43))(x)
model = Model(inputs=inputs, outputs=predictions, name='mnist_model')

model.compile(loss='categorical_crossentropy',
            optimizer=optimizers.Adam(),
            metrics=['accuracy'])

history = model.fit(x_train, y_train,
                    batch_size=5,
                    epochs=10)
model.summary()

test_scores = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', test_scores[0])
print('Test accuracy:', test_scores[1])

weights = []
for layer in model.layers:
    weights = layer.get_weights()
    np.save("../weightBias/"+layer.name,weights)

Each layer weights are different though initial weights are initialized with the same random distribution.

Please point out why I am getting different outputs for the same model.

TensorFlow accuracy:

Step 1, Minibatch Loss= 68458.3750, Training Accuracy= 0.800
Step 10, Minibatch Loss= 451470.5625, Training Accuracy= 0.200
Step 20, Minibatch Loss= 582654.3750, Training Accuracy= 0.200
Step 30, Minibatch Loss= 185989.4219, Training Accuracy= 0.400
Step 1, Minibatch Loss= 161536.2188, Training Accuracy= 0.600
Step 10, Minibatch Loss= 286743.1875, Training Accuracy= 0.400
Step 20, Minibatch Loss= 206555.5000, Training Accuracy= 0.600
Step 30, Minibatch Loss= 201791.6250, Training Accuracy= 0.800
Step 1, Minibatch Loss= 212732.3438, Training Accuracy= 0.600
Step 10, Minibatch Loss= 108325.5469, Training Accuracy= 0.800
Step 20, Minibatch Loss= 80389.1641, Training Accuracy= 0.800
Step 30, Minibatch Loss= 34269.3750, Training Accuracy= 0.800
Step 1, Minibatch Loss= 134766.6406, Training Accuracy= 0.400
Step 10, Minibatch Loss= 166675.9531, Training Accuracy= 0.400
Step 20, Minibatch Loss= 66996.0938, Training Accuracy= 0.600
Step 30, Minibatch Loss= 29773.1660, Training Accuracy= 0.800
Step 1, Minibatch Loss= 33207.5234, Training Accuracy= 0.600
Step 10, Minibatch Loss= 154839.6250, Training Accuracy= 0.800
Step 20, Minibatch Loss= 23777.3125, Training Accuracy= 0.800
Step 30, Minibatch Loss= 57576.0742, Training Accuracy= 0.400
Optimization Finished!
Testing Accuracy: 0.2

Keras Accuracy:

156/156 [==============================] - 6s 41ms/step - loss: 13.0185 - acc: 0.1923 
Epoch 2/10
156/156 [==============================] - 3s 18ms/step - loss: 12.9151 - acc: 0.1987
Epoch 3/10
156/156 [==============================] - 3s 18ms/step - loss: 13.1218 - acc: 0.1859
Epoch 4/10
156/156 [==============================] - 3s 18ms/step - loss: 12.9151 - acc: 0.1987
Epoch 5/10
156/156 [==============================] - 3s 19ms/step - loss: 13.1218 - acc: 0.1859
Epoch 6/10
156/156 [==============================] - 3s 19ms/step - loss: 12.9151 - acc: 0.1987
Epoch 7/10
156/156 [==============================] - 3s 19ms/step - loss: 12.8118 - acc: 0.2051
Epoch 8/10
156/156 [==============================] - 3s 19ms/step - loss: 12.8118 - acc: 0.2051
Epoch 9/10
156/156 [==============================] - 3s 19ms/step - loss: 12.8118 - acc: 0.2051
Epoch 10/10
156/156 [==============================] - 3s 19ms/step - loss: 12.9151 - acc: 0.1987
$\endgroup$

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.