I am trying to train a CNN, using the MNIST dataset (which I perform data augmentation on), to classify numbers on a sudoku grid from 0-9.
While mostly successful, my network seems to get confused between 3s and 8s, and 1s and 7s because of how similar they look. This is unacceptable, however, since incorrect classification will make solving the sudoku problem impossible.
I am using the ResNet50 pre-trained model as my convolutional base.
Is there any way to more harshly penalise mis-classification of 3s and 8s, or 1s and 7s during the training process? I did find a link which seems to propose a solution - https://github.com/keras-team/keras/issues/2115 - but I am quite new to TensorFlow/Keras, and don't quite understand the code given.
I would very much appreciate if anyone could either explain the proposed solution on the link above, or suggest improvements to my network/code (which I have added below).
THE NETWORK:
import cv2 as cv
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow_datasets as tfds
tfds.disable_progress_bar()
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
# To make the model work with grayscale images, we need to make them APPEAR to be RGB. The easiest way is to repeat the image array 3 times on a new dimension.
# Because we will have the same image over all 3 channels, the performance of the model should be the same as it was on RGB images.
train_images, test_images = np.repeat(train_images[..., np.newaxis], 3, -1), np.repeat(test_images[...,np.newaxis], 3, -1)
# Normalize pixel values to be between 0 and 1
train_images, test_images = train_images / 255.0, test_images / 255.0
# Resize the input shape because ResNet50 can take the input image having height, width as multiples of 32 and 3 as channel width
train_images, test_images = tf.image.resize(train_images, [32,32]), tf.image.resize(test_images, [32,32])
assert train_images.shape == (60000, 32, 32, 3) # All images are 28x28 - no resizing needed
assert test_images.shape == (10000, 32, 32, 3)
assert train_labels.shape == (60000,) # Labels - numbers from 0 to 9
assert test_labels.shape == (10000,) # Labels - numbers from 0 to
# Data augmentation parameters
rotation_range_val = 10 # rotation
width_shift_val = 0.1 # horizontal shift
height_shift_val = 0.1 # vertical shift
zoom_range_val=[0.8,1.3] # zoom
train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(
rotation_range = rotation_range_val,
width_shift_range = width_shift_val,
height_shift_range = height_shift_val,
zoom_range = zoom_range_val,
)
train_datagen.fit(train_images)
# Pretrained convolutional base
base_model = tf.keras.applications.ResNet50(input_shape = (32,32,3), include_top=False, weights='imagenet')
# Freezing the base
base_model.trainable = False
# Adding dense layers on top
classifier_model = tf.keras.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(128, activation='relu'), # hidden layer
tf.keras.layers.Dense(10) # output layer
])
# Combine base and classifier
model = tf.keras.Sequential([
base_model,
classifier_model
])
# Compile the model
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
# Fits the model on batches with real-time data augmentation
history = model.fit(train_datagen.flow(train_images, train_labels, batch_size=64), epochs=20, verbose=2) # If unspecified, batch size defaults to 32.
plt.plot(history.history['accuracy'], label='accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.ylim([0.5, 1])
plt.legend(loc='lower right')
plt.show()
test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print(test_acc)
model.save('C:\Programming 2022-23\OpenCV\Sudoku Solver\ResNet50_model_with_augmentation')
HOW I AM USING THE NETWORK TO CLASSIFY:
import cv2 as cv
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow_datasets as tfds
tfds.disable_progress_bar()
localised_grid = np.load("C:\Programming 2022-23\OpenCV\Sudoku Solver\Localised sudoku grid.npy")
# Visualising the gridlines
output_visualise = np.copy(localised_grid)
for i in range (0,500,50):
cv.line(output_visualise, (i,0), (i,450),(255,0,0), thickness = 3) # Drawing vertical lines
cv.line(output_visualise, (0,i), (450,i),(255,0,0), thickness = 3) # Drawing horizontal lines
#cv.imshow("Gridlines", output_visualise)
# MNIST dataset contains black and white images, so we threshold the images
localised_grid_gray = cv.cvtColor(localised_grid, cv.COLOR_BGR2GRAY)
adaptive_thresh = cv.adaptiveThreshold(localised_grid_gray, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY_INV, 11, 2)
for i in range (0,2):
adaptive_thresh = cv.medianBlur(adaptive_thresh, 3)
cv.imshow("Thresholded", adaptive_thresh)
sudoku_grid_stored = np.empty((9,9,46,46)).astype("uint8")
for y in range(0,9):
for x in range(0,9):
cropped_digit = adaptive_thresh[50*y+2:(50*y)+48,50*x+2:(50*x)+48].astype("uint8") # Crop slightly to remove any sudoku grid outlines that may exist
sudoku_grid_stored[y][x] = cropped_digit
# cv.imshow("7", sudoku_grid_stored[2][0]) # 7 lies in 3rd row, 1st column (zero indexed)
# Inputs to CNN require must have three colour channels
sudoku_grid_stored = np.repeat(sudoku_grid_stored[..., np.newaxis], 3, -1)
# Inputs to CNN must be normalised between 0 and 1, since this is how it was trained
sudoku_grid_stored = sudoku_grid_stored/255.0
# Inputs to CNN must be 32x32, since this is how it was trained
sudoku_grid_resized= np.empty((9,9,32,32,3))
for j in range(0,9):
for i in range (0,9):
sudoku_grid_resized[j][i] = cv.resize(sudoku_grid_stored[j][i], (32,32))
# Load trained CNN
CNN_model = tf.keras.models.load_model('C:\Programming 2022-23\OpenCV\Sudoku Solver\ResNet50_model_with_augmentation')
test = sudoku_grid_resized[0][0]
test = np.expand_dims(test,axis=0)
prediction = np.argmax(CNN_model.predict(test))
print(prediction)
RESULT OF THRESHOLDING (I.E. INPUTS TO CNN) - for example, the top left value of 3 is being read as 8.
