I am trying to train a CNN, using the MNIST dataset (which I perform data augmentation on), to classify numbers on a sudoku grid from 0-9.
While mostly successful, my network seems to get confused between 3s and 8s, and 1s and 7s because of how similar they look. This is unacceptable, however, since incorrect classification will make solving the sudoku problem impossible.
I am using the ResNet50 pre-trained model as my convolutional base.
Is there any way to more harshly penalise mis-classification of 3s and 8s, or 1s and 7s during the training process? I did find a link which seems to propose a solution - https://github.com/keras-team/keras/issues/2115 - but I am quite new to TensorFlow/Keras, and don't quite understand the code given.
I would very much appreciate if anyone could either explain the proposed solution on the link above, or suggest improvements to my network/code (which I have added below).
import cv2 as cv import numpy as np import matplotlib.pyplot as plt import tensorflow as tf import tensorflow_datasets as tfds tfds.disable_progress_bar() (train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data() # To make the model work with grayscale images, we need to make them APPEAR to be RGB. The easiest way is to repeat the image array 3 times on a new dimension. # Because we will have the same image over all 3 channels, the performance of the model should be the same as it was on RGB images. train_images, test_images = np.repeat(train_images[..., np.newaxis], 3, -1), np.repeat(test_images[...,np.newaxis], 3, -1) # Normalize pixel values to be between 0 and 1 train_images, test_images = train_images / 255.0, test_images / 255.0 # Resize the input shape because ResNet50 can take the input image having height, width as multiples of 32 and 3 as channel width train_images, test_images = tf.image.resize(train_images, [32,32]), tf.image.resize(test_images, [32,32]) assert train_images.shape == (60000, 32, 32, 3) # All images are 28x28 - no resizing needed assert test_images.shape == (10000, 32, 32, 3) assert train_labels.shape == (60000,) # Labels - numbers from 0 to 9 assert test_labels.shape == (10000,) # Labels - numbers from 0 to # Data augmentation parameters rotation_range_val = 10 # rotation width_shift_val = 0.1 # horizontal shift height_shift_val = 0.1 # vertical shift zoom_range_val=[0.8,1.3] # zoom train_datagen = tf.keras.preprocessing.image.ImageDataGenerator( rotation_range = rotation_range_val, width_shift_range = width_shift_val, height_shift_range = height_shift_val, zoom_range = zoom_range_val, ) train_datagen.fit(train_images) # Pretrained convolutional base base_model = tf.keras.applications.ResNet50(input_shape = (32,32,3), include_top=False, weights='imagenet') # Freezing the base base_model.trainable = False # Adding dense layers on top classifier_model = tf.keras.Sequential([ tf.keras.layers.Flatten(), tf.keras.layers.Dense(128, activation='relu'), # hidden layer tf.keras.layers.Dense(10) # output layer ]) # Combine base and classifier model = tf.keras.Sequential([ base_model, classifier_model ]) # Compile the model model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) # Fits the model on batches with real-time data augmentation history = model.fit(train_datagen.flow(train_images, train_labels, batch_size=64), epochs=20, verbose=2) # If unspecified, batch size defaults to 32. plt.plot(history.history['accuracy'], label='accuracy') plt.xlabel('Epoch') plt.ylabel('Accuracy') plt.ylim([0.5, 1]) plt.legend(loc='lower right') plt.show() test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2) print(test_acc) model.save('C:\Programming 2022-23\OpenCV\Sudoku Solver\ResNet50_model_with_augmentation')
HOW I AM USING THE NETWORK TO CLASSIFY:
import cv2 as cv import numpy as np import matplotlib.pyplot as plt import tensorflow as tf import tensorflow_datasets as tfds tfds.disable_progress_bar() localised_grid = np.load("C:\Programming 2022-23\OpenCV\Sudoku Solver\Localised sudoku grid.npy") # Visualising the gridlines output_visualise = np.copy(localised_grid) for i in range (0,500,50): cv.line(output_visualise, (i,0), (i,450),(255,0,0), thickness = 3) # Drawing vertical lines cv.line(output_visualise, (0,i), (450,i),(255,0,0), thickness = 3) # Drawing horizontal lines #cv.imshow("Gridlines", output_visualise) # MNIST dataset contains black and white images, so we threshold the images localised_grid_gray = cv.cvtColor(localised_grid, cv.COLOR_BGR2GRAY) adaptive_thresh = cv.adaptiveThreshold(localised_grid_gray, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY_INV, 11, 2) for i in range (0,2): adaptive_thresh = cv.medianBlur(adaptive_thresh, 3) cv.imshow("Thresholded", adaptive_thresh) sudoku_grid_stored = np.empty((9,9,46,46)).astype("uint8") for y in range(0,9): for x in range(0,9): cropped_digit = adaptive_thresh[50*y+2:(50*y)+48,50*x+2:(50*x)+48].astype("uint8") # Crop slightly to remove any sudoku grid outlines that may exist sudoku_grid_stored[y][x] = cropped_digit # cv.imshow("7", sudoku_grid_stored) # 7 lies in 3rd row, 1st column (zero indexed) # Inputs to CNN require must have three colour channels sudoku_grid_stored = np.repeat(sudoku_grid_stored[..., np.newaxis], 3, -1) # Inputs to CNN must be normalised between 0 and 1, since this is how it was trained sudoku_grid_stored = sudoku_grid_stored/255.0 # Inputs to CNN must be 32x32, since this is how it was trained sudoku_grid_resized= np.empty((9,9,32,32,3)) for j in range(0,9): for i in range (0,9): sudoku_grid_resized[j][i] = cv.resize(sudoku_grid_stored[j][i], (32,32)) # Load trained CNN CNN_model = tf.keras.models.load_model('C:\Programming 2022-23\OpenCV\Sudoku Solver\ResNet50_model_with_augmentation') test = sudoku_grid_resized test = np.expand_dims(test,axis=0) prediction = np.argmax(CNN_model.predict(test)) print(prediction)