0
$\begingroup$

I have this model which takes 9000 images in a dataset containing 96 categories of traffic signs, each category has more or less the same number of images (about 50). This is the model I made but somehow the predictions are really bad even if the validation accuracy is really high (99%). I can't figure it out what's wrong. I read some possibilities are: overfitting, cnn is too big for the dataset I use, I train on the same data I use to validate the model. How can I understand where I am failing at?

JSON_PATH = "/Users/user/Documents/ML Projects/classname.json"
DATASET_PATH = "/Users/user/Documents/ML Projects/Dataset"


CLASSNAME_SIZE = 96
IMG_SIZE = 48

with open(JSON_PATH) as classnameJSON:
    CLASSNAME = json.loads(classnameJSON.read())

trainingData = []
X = []
Y = []
X_val = []
Y_val = []


def loadTrainingData():
    for instance in range(CLASSNAME_SIZE):
        joinedPath = os.path.join(DATASET_PATH, str(instance))
        label = str(instance)
        for img in os.listdir(joinedPath):
            try:
                img_array = cv2.imread(os.path.join(joinedPath,img), cv2.COLOR_BGR2RGB)
                new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
                trainingData.append([new_array, label])
            except Exception as err:
                pass

loadTrainingData()
print(len(trainingData))

def distributeTrainingData():
    for img, label in trainingData:
        X.append(img)
        X_val.append(img)
        Y.append(label)
        Y_val.append(label)

distributeTrainingData()
print("distributing data")

X = np.array(X, dtype='float32')
Y = np.array(Y, dtype='float32')
X_val = np.array(X_val, dtype='float32')
Y_val = np.array(Y_val, dtype='float32')
print(len(X))
print(len(Y))

def cnn_model():
    model = Sequential()
    model.add(Conv2D(32, (3, 3), padding='same',
                     activation='relu'))
    model.add(Conv2D(32, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
    model.add(Conv2D(128, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(keras.layers.InputLayer(input_shape=(X.shape[1])))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.25))
    model.add(Dense(CLASSNAME_SIZE, activation='softmax'))
    return model

#model = cnn_model()
model = keras.models.load_model('traffic_signs.model')


model.compile(loss='sparse_categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

model.fit(X, Y, batch_size=32, epochs=30, validation_data=(X_val,Y_val), shuffle=True)
model.save('traffic_signs.model')```
$\endgroup$
3
  • $\begingroup$ I think you overfitting, for that you must change training set. $\endgroup$
    – koprotk
    Dec 12 '20 at 14:29
  • $\begingroup$ Didnt you mean the test set? If you see the code both the training and the test set are the same, this might be the problem but not sure $\endgroup$
    – karalis1
    Dec 12 '20 at 15:22
  • $\begingroup$ This may be a case of data leakage. Check if you are inadvertently including in the training set some data from the validation set. This would explain why you get a high validation accuracy but a low test accuracy. Also, check if accuracy is an appropriate performance measure; for instance, if the test data distribution is not balanced, you may be failing many instances of a specific category. Also, your training data seems quite small for such a network capacity. $\endgroup$
    – noe
    Dec 12 '20 at 15:44

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.