0
$\begingroup$

I got this R Keras model from GitHub that performs really well. GitHub repo

library(keras)
library(tidyverse)
# Data Preparation -----------------------------------------------------

batch_size <- 100
num_classes <- 10
epochs <- 5

# Input image dimensions
img_rows <- 28
img_cols <- 28

# The data, shuffled and split between train and test sets
mnist <- dataset_mnist()
x_train <- mnist$train$x
y_train <- mnist$train$y
x_test <- mnist$test$x
y_test <- mnist$test$y

# Redefine  dimension of train/test inputs
x_train <- array_reshape(x_train, c(nrow(x_train), img_rows, img_cols, 1))
x_test <- array_reshape(x_test, c(nrow(x_test), img_rows, img_cols, 1))
input_shape <- c(img_rows, img_cols, 1)

# Transform RGB values into [0,1] range
x_train <- x_train / 255
x_test <- x_test / 255

# Convert class vectors to binary class matrices
y_train <- to_categorical(y_train, num_classes)
y_test <- to_categorical(y_test, num_classes)

# Define Model -----------------------------------------------------------

# SimpNetV2
# https://github.com/Coderx7/SimpNet/blob/master/SimpNetV2/Logs/MNIST/caffe_99.75.log
# "MNIST_SimpleNet_GP_13L_drpall_5Mil_66_maxdrp"

normal_krnl <- initializer_random_normal(stddev = 0.01)
model <-  keras_model_sequential() %>%
  ### Block 1
  ## Conv 1_0 (conv 1)
  layer_conv_2d(filters = 66, kernel_size = c(3,3), padding = "same",
                kernel_initializer = "glorot_uniform", # aka Xavier
                input_shape = input_shape) %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%

  ### Block 2
  ## Conv 2_0 (conv 2)
  layer_conv_2d(filters = 64, kernel_size = c(3,3), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%
  ## Conv 2_1 (conv 3)
  layer_conv_2d(filters = 64, kernel_size = c(3,3), padding = "same",
                kernel_initializer = normal_krnl) %>% # gaussian Kernel
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%
  ## Conv 2_2 (conv 4)
  layer_conv_2d(filters = 64, kernel_size = c(3,3), padding = "same",
                kernel_initializer = normal_krnl) %>% # Gaussian
  layer_batch_normalization(momentum = 0.95,
                            scale = TRUE, gamma_regularizer = "l2") %>% # <- ??
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%

  ### Block 3
  ## Conv 3_0 (conv 5)
  layer_conv_2d(filters = 96, kernel_size = c(3,3), padding = "same",
                kernel_initializer = normal_krnl) %>% # Gaussian
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_max_pooling_2d(pool_size = c(2, 2), stride = 2) %>%
  layer_dropout(rate = 0.2) %>%

  ### Block 4
  ## Conv 4_0 (conv 6)
  layer_conv_2d(filters = 96, kernel_size = c(3,3), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%
  ## Conv 4_1 (conv 7)
  layer_conv_2d(filters = 96, kernel_size = c(3,3), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%
  ## Conv 4_3 (conv 8)
  layer_conv_2d(filters = 96, kernel_size = c(3,3), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%
  ## Conv 4_4 (conv 9)
  layer_conv_2d(filters = 96, kernel_size = c(3,3), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%

  ### Block 5
  ## Conv 5_0 (conv 10)
  layer_conv_2d(filters = 144, kernel_size = c(3,3), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_max_pooling_2d(pool_size = c(2, 2), stride = 2) %>%
  layer_dropout(rate = 0.2) %>%

  ### Block 6
  ## Conv 6_0 (conv 11)
  layer_conv_2d(filters = 144, kernel_size = c(1,1), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%

  ### Block 7
  ## conv 7_0 (conv 12)
  layer_conv_2d(filters = 178, kernel_size = c(1,1), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_dropout(rate = 0.2) %>%

  ### Block 8
  ## Conv 8_0 (conv 13)
  layer_conv_2d(filters = 216, kernel_size = c(3,3), padding = "same") %>%
  layer_batch_normalization(momentum = 0.95, scale = TRUE) %>%
  layer_activation("relu") %>%
  layer_global_max_pooling_2d() %>%
  layer_dropout(rate = 0.2) %>%
  ### Inference
  layer_dense(units = num_classes, activation = 'softmax')

# Compile model
model %>% compile(
  loss = loss_categorical_crossentropy,
  optimizer = optimizer_adadelta(),
  metrics = c('accuracy')
)
# Train model
model %>% fit(
  x_train, y_train,
  batch_size = batch_size,
  epochs = epochs
)
scores <- model %>% evaluate(
  x_test, y_test, verbose = 1
)

# Output metrics
cat('Test loss:', scores[[1]], '\n')
cat('Test accuracy:', scores[[2]], '\n')

I went through line by line and remade it in Python Keras:

import os
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import regularizers, optimizers
from tensorflow.keras.layers import Conv2D, MaxPooling2D, GlobalMaxPooling2D
from tensorflow.keras.initializers import glorot_uniform, RandomNormal, Zeros
from tensorflow.keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['KMP_WARNINGS'] = '0'
tf.logging.set_verbosity(tf.logging.ERROR)

# Data Preparation -----------------------------------------------------

batch_size = 100
num_classes = 10
epochs = 5

# Input image dimensions
img_rows = 28
img_cols = 28

mnist = keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()

# Redefine  dimension of train/test inputs
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)  # potential problem
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)     # potential problem
input_shape = (img_rows, img_cols, 1)

# Transform RGB values into [0,1] range
x_train = x_train.astype("float32")
x_test = x_test.astype("float32")

x_train /= 255
x_test /= 255

# Convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)


# Define Model -----------------------------------------------------------

# SimpNetV2
# https://github.com/Coderx7/SimpNet/blob/master/SimpNetV2/Logs/MNIST/caffe_99.75.log
# "MNIST_SimpleNet_GP_13L_drpall_5Mil_66_maxdrp"

normal_krnl = keras.initializers.RandomNormal(stddev = 0.01)
model =  keras.models.Sequential()
  ### Block 1
  ## Conv 1_0 (conv 1)
model.add(Conv2D(66, (3,3), padding='same', kernel_initializer=glorot_uniform(), input_shape=input_shape))
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))

  ### Block 2
  ## Conv 2_0 (conv 2)
model.add(Conv2D(64, (3,3), padding='same')) # maybe add a kernel initializer here
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))
  ## Conv 2_1 (conv 3)
model.add(Conv2D(64, (3,3), padding='same', kernel_initializer=normal_krnl))
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))
  ## Conv 2_2 (conv 4)
model.add(Conv2D(64, (3,3), padding='same', kernel_initializer=normal_krnl))
model.add(BatchNormalization(momentum=0.95, scale=True, gamma_regularizer='l2'))
model.add(Activation('relu'))
model.add(Dropout(0.2))

  ### Block 3
  ## Conv 3_0 (conv 5)
model.add(Conv2D(96, (3,3), padding='same', kernel_initializer=normal_krnl))
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
model.add(Dropout(0.2))

  ### Block 4
  ## Conv 4_0 (conv 6)
model.add(Conv2D(96, (3,3), padding='same')) # inititalizer?
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))
  ## Conv 4_1 (conv 7)
model.add(Conv2D(96, (3,3), padding='same')) # inititalizer?
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))
  ## Conv 4_3 (conv 8)
model.add(Conv2D(96, (3,3), padding='same')) # inititalizer?
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))
  ## Conv 4_4 (conv 9)
model.add(Conv2D(96, (3,3), padding='same')) # inititalizer?
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))

  ### Block 5
  ## Conv 5_0 (conv 10)
model.add(Conv2D(144, (3,3), padding='same'))
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
model.add(Dropout(0.2))

  ### Block 6
  ## Conv 6_0 (conv 11)
model.add(Conv2D(144, (1,1), padding='same'))
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))

  ### Block 7
  ## conv 7_0 (conv 12)
model.add(Conv2D(178, (1,1), padding='same'))
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.2))

  ### Block 8
  ## Conv 8_0 (conv 13)
model.add(Conv2D(216, (3,3), padding='same'))
model.add(BatchNormalization(momentum=0.95, scale=True))
model.add(Activation('relu'))
model.add(GlobalMaxPooling2D())
model.add(Dropout(0.2))
  ### Inference
model.add(Dense(units=num_classes, activation='softmax'))

# Compile model
ada_delta = keras.optimizers.Adadelta()

model.compile(loss='categorical_crossentropy',
              optimizer=ada_delta,
              metrics=['accuracy'])
# Train model
model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs)

val_loss, val_acc = model.evaluate(x_test, y_test, verbose=1)

# Output metrics
print()
print('LOSS')
print(val_loss)
print('ACC')
print(val_acc)

The Python version performs incredibly bad compared to the R version. I ran it for 10 epochs and the train accuracy was around 12%. 10 epochs on the R version lead to 80%+ train accuracy. I went through many times to make sure all of my parameters and method calls are identical and I have not found any differences. Calling model.summary(), (summary(model) in R) print identical results. I have tried changing the random seeds in the Python version but this did not help. A similar question on StackOverflow that basically remained unanswered can be found here. I post a similar question on here because the data science stack exchange community is more likely to know the solution.

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.