6
$\begingroup$

I've used transfer learning on Inception V3 with ImageNet weights on Keras with Tensorflow backend on python 2.7 to create an image classifier. I first extracted and saved the bottleneck features from Inception and used them to train a fully connected layer. This FC gave me around 86% training accuracy. I then 'fine tuned' the model by sticking the trained FC layers on top of a topless Inception V3 and retrained along with the top two convolutional blocks with SGD, a low learning rate and high momentum. I also used Image Augmentation on the training images, but only during fine tuning stage. Instead of accuracy improving, this model hardly gives me 60% accuracy and performs worse than the bare transfer learning model.

1.Why is this happening?

Optional: 2.What can I do to improve my accuracy in 'general' on this model? The images I am trying to classify are from the BreakHis breast tumor database and is quite a difficult classification task. Is Transfer learning even appropriate for this task?

Code for Transfer learning with bottleneck features:

import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dropout, Dense, GlobalAveragePooling2D
from keras.applications.inception_v3 import InceptionV3,preprocess_input
from keras.utils.np_utils import to_categorical
from keras.callbacks import ModelCheckpoint, EarlyStopping
import math

img_width, img_height = (299, 299)
weight_path = 'bottleneck_fc_model.h5'
train_dir = 'Cancer_Data/Train'
validation_dir = 'Cancer_Data/Validate'
epochs = 150
batch_size = 128



def train_top_model():
    datagen_top = ImageDataGenerator(preprocessing_function=preprocess_input)
    generator_top = datagen_top.flow_from_directory(
    train_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='categorical',
    shuffle=False)

nb_classes = len(generator_top.class_indices)
np.save('class_indices.npy', generator_top.class_indices)

train_data = np.load('bottleneck_feature_train.npy')
train_labels = to_categorical(generator_top.classes, num_classes=nb_classes)

generator_top2 = datagen_top.flow_from_directory(
    validation_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='categorical',
    shuffle=False)

validation_data = np.load('bottleneck_feature_validation.npy')
validation_labels = to_categorical(generator_top2.classes, num_classes=nb_classes)

model = Sequential()
model.add(GlobalAveragePooling2D(input_shape=train_data.shape[1:]))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes, activation='softmax'))

model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

callbacks_list=[ModelCheckpoint(weight_path, monitor='val_acc', verbose=1, save_best_only=True),EarlyStopping(monitor='val_acc',patience=10,verbose=0)]



model.fit(train_data, train_labels, epochs=epochs, batch_size=batch_size,
          validation_data=(validation_data, validation_labels), callbacks=callbacks_list)

(eval_loss, eval_accuracy) = model.evaluate(
    validation_data, validation_labels, batch_size=batch_size, verbose=1)

print("[INFO] accuracy: {:.2f}%".format(eval_accuracy * 100))
print("[INFO] Loss: {}".format(eval_loss))


train_top_model()

Code for Fine Tuning:

from keras.applications.inception_v3 import InceptionV3, preprocess_input
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.regularizers import l1_l2
from keras.models import Sequential, Model, model_from_json
from keras.layers import Dropout, GlobalAveragePooling2D, Dense
from keras.callbacks import ModelCheckpoint, EarlyStopping
import PIL
import math


weight_path = 'fine_tuned_weights.h5'
top_model_weight_path = 'top_model.h5'

img_width, img_height = (229, 229)

train_dir = 'Cancer_Data/Train'
validation_dir = 'Cancer_Data/Validate'

epochs = 150

batch_size = 128

nb_train_samples = 6454
nb_validation_samples = 1464

base_model =InceptionV3(weights= 'imagenet', include_top= False, input_shape=(229,229,3))
print "Model Loaded."

top_model= Sequential()
top_model.add(GlobalAveragePooling2D(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(1024, activation='relu'))
top_model.add(Dense(8, activation= 'softmax'))

top_model.load_weights(top_model_weight_path)

model= Model(inputs= base_model.input, outputs= top_model(base_model.output))

for layer in model.layers[:172]:
    layer.trainable=False
for layer in model.layers[172:]:
    layer.trainable=True

model.compile(optimizer=optimizers.SGD(lr=0.0001, momentum=0.9),
          loss='categorical_crossentropy', metrics=['accuracy'])

train_datagen= ImageDataGenerator(
    preprocessing_function=preprocess_input,
    rotation_range=20,
    shear_range=0.3,
    zoom_range=0.3,
    horizontal_flip=True,
    vertical_flip=True)

test_datagen= ImageDataGenerator(preprocessing_function=preprocess_input)

train_generator= train_datagen.flow_from_directory(
    train_dir,
    target_size=(img_height,img_width),
    batch_size=batch_size,
    class_mode='categorical')

validation_generator= test_datagen.flow_from_directory(
    validation_dir,
    target_size=(img_height,img_width),
    batch_size=batch_size,
    class_mode='categorical')

callbacks_list=[ModelCheckpoint(weight_path, monitor='val_acc',verbose=1,save_best_only=True),
            EarlyStopping(monitor='val_acc',patience=10,verbose=0)]

model_json = model.to_json()
with open("fine_tuned_model.json", "w") as json_file:
    json_file.write(model_json)

model.fit_generator(
    train_generator,
    steps_per_epoch = int(math.ceil(nb_train_samples / batch_size)),
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps = int(math.ceil(nb_validation_samples / batch_size)), callbacks=callbacks_list)
$\endgroup$
1
  • 3
    $\begingroup$ One missing thing in your Code for Fine Tuning was two-step training. You should do fine-tuning after you train the top layer with all lower layers frozen (set trainable = False). Then you should un-freeze (set trainable = True) some of the top convolutional layers and do fine-tuning with smaller learning rates. $\endgroup$
    – pronot
    May 20, 2018 at 21:00

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.