Fine tuning accuracy lower than Raw Transfer Learning Accuracy

Question

I've used transfer learning on Inception V3 with ImageNet weights on Keras with Tensorflow backend on python 2.7 to create an image classifier. I first extracted and saved the bottleneck features from Inception and used them to train a fully connected layer. This FC gave me around 86% training accuracy. I then 'fine tuned' the model by sticking the trained FC layers on top of a topless Inception V3 and retrained along with the top two convolutional blocks with SGD, a low learning rate and high momentum. I also used Image Augmentation on the training images, but only during fine tuning stage. Instead of accuracy improving, this model hardly gives me 60% accuracy and performs worse than the bare transfer learning model.

1.Why is this happening?

Optional: 2.What can I do to improve my accuracy in 'general' on this model? The images I am trying to classify are from the BreakHis breast tumor database and is quite a difficult classification task. Is Transfer learning even appropriate for this task?

Code for Transfer learning with bottleneck features:

import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dropout, Dense, GlobalAveragePooling2D
from keras.applications.inception_v3 import InceptionV3,preprocess_input
from keras.utils.np_utils import to_categorical
from keras.callbacks import ModelCheckpoint, EarlyStopping
import math

img_width, img_height = (299, 299)
weight_path = 'bottleneck_fc_model.h5'
train_dir = 'Cancer_Data/Train'
validation_dir = 'Cancer_Data/Validate'
epochs = 150
batch_size = 128



def train_top_model():
    datagen_top = ImageDataGenerator(preprocessing_function=preprocess_input)
    generator_top = datagen_top.flow_from_directory(
    train_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='categorical',
    shuffle=False)

nb_classes = len(generator_top.class_indices)
np.save('class_indices.npy', generator_top.class_indices)

train_data = np.load('bottleneck_feature_train.npy')
train_labels = to_categorical(generator_top.classes, num_classes=nb_classes)

generator_top2 = datagen_top.flow_from_directory(
    validation_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='categorical',
    shuffle=False)

validation_data = np.load('bottleneck_feature_validation.npy')
validation_labels = to_categorical(generator_top2.classes, num_classes=nb_classes)

model = Sequential()
model.add(GlobalAveragePooling2D(input_shape=train_data.shape[1:]))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes, activation='softmax'))

model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

callbacks_list=[ModelCheckpoint(weight_path, monitor='val_acc', verbose=1, save_best_only=True),EarlyStopping(monitor='val_acc',patience=10,verbose=0)]



model.fit(train_data, train_labels, epochs=epochs, batch_size=batch_size,
          validation_data=(validation_data, validation_labels), callbacks=callbacks_list)

(eval_loss, eval_accuracy) = model.evaluate(
    validation_data, validation_labels, batch_size=batch_size, verbose=1)

print("[INFO] accuracy: {:.2f}%".format(eval_accuracy * 100))
print("[INFO] Loss: {}".format(eval_loss))


train_top_model()

Code for Fine Tuning:

from keras.applications.inception_v3 import InceptionV3, preprocess_input
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.regularizers import l1_l2
from keras.models import Sequential, Model, model_from_json
from keras.layers import Dropout, GlobalAveragePooling2D, Dense
from keras.callbacks import ModelCheckpoint, EarlyStopping
import PIL
import math


weight_path = 'fine_tuned_weights.h5'
top_model_weight_path = 'top_model.h5'

img_width, img_height = (229, 229)

train_dir = 'Cancer_Data/Train'
validation_dir = 'Cancer_Data/Validate'

epochs = 150

batch_size = 128

nb_train_samples = 6454
nb_validation_samples = 1464

base_model =InceptionV3(weights= 'imagenet', include_top= False, input_shape=(229,229,3))
print "Model Loaded."

top_model= Sequential()
top_model.add(GlobalAveragePooling2D(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(1024, activation='relu'))
top_model.add(Dense(8, activation= 'softmax'))

top_model.load_weights(top_model_weight_path)

model= Model(inputs= base_model.input, outputs= top_model(base_model.output))

for layer in model.layers[:172]:
    layer.trainable=False
for layer in model.layers[172:]:
    layer.trainable=True

model.compile(optimizer=optimizers.SGD(lr=0.0001, momentum=0.9),
          loss='categorical_crossentropy', metrics=['accuracy'])

train_datagen= ImageDataGenerator(
    preprocessing_function=preprocess_input,
    rotation_range=20,
    shear_range=0.3,
    zoom_range=0.3,
    horizontal_flip=True,
    vertical_flip=True)

test_datagen= ImageDataGenerator(preprocessing_function=preprocess_input)

train_generator= train_datagen.flow_from_directory(
    train_dir,
    target_size=(img_height,img_width),
    batch_size=batch_size,
    class_mode='categorical')

validation_generator= test_datagen.flow_from_directory(
    validation_dir,
    target_size=(img_height,img_width),
    batch_size=batch_size,
    class_mode='categorical')

callbacks_list=[ModelCheckpoint(weight_path, monitor='val_acc',verbose=1,save_best_only=True),
            EarlyStopping(monitor='val_acc',patience=10,verbose=0)]

model_json = model.to_json()
with open("fine_tuned_model.json", "w") as json_file:
    json_file.write(model_json)

model.fit_generator(
    train_generator,
    steps_per_epoch = int(math.ceil(nb_train_samples / batch_size)),
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps = int(math.ceil(nb_validation_samples / batch_size)), callbacks=callbacks_list)