0
$\begingroup$

I am new to Machine Learning, CNN and Caffe and I have an issue I would be very happy to solve. As part of a University project I must use a Machine Learning method to classify images into 3 classes. I use Caffe on Python for the task.

LONG STORY SHORT (Code is attached later on):

After finishing the training stage, which resulted in(from log file):

Test net output #0: accuracy = 1
Test net output #1: loss = 0.000219849 (* 1 = 0.000219849 loss)
Optimization Done.
Optimization Done.

I tried to predict for a test set and received a constant prediction for all images. I have tried to check my code for correct functionality, and found that even when i use the validation set for predicting- I get a constant prediction. How can that be? shouldn't my net be able to classify those with great accuracy?

CODE

TRAINING:

 import  os

 import glob

 from datetime import datetime

 from config import cifar10_config as config

 import argparse
 ap = argparse.ArgumentParser()
 ap.add_argument("-l", "--logfile", required=True, help="logfile")
 ap.add_argument("-s", "--solver",  required=True, help="solver") 

args = vars(ap.parse_args())
LOGFILE = args["logfile"]
SOLVER  = args["solver"]

CAFFE_TRAINING = config.CAFFE_TOOLS_DIR + "/bin/caffe.bin train "      #i.e. "/caffe/Caffe-SSD-Ristretto/distribute/bin/caffe.bin train "

print("TRAINING WITH CAFFE")

caffe_solver  = SOLVER

caffe_logfile = LOGFILE

caffe_command = CAFFE_TRAINING + ' --solver ' + caffe_solver + ' 2>&1 | tee ' + caffe_logfile

startTime1 = datetime.now()

os.system(caffe_command)

endTime1 = datetime.now()

diff1 = endTime1 - startTime1

print("\n")

print("Elapsed time for Caffe training (s): ", diff1.total_seconds())

print("\n")



**PREDICT**(This version is the one in which I load the validation set images for prediction)

import os

import glob

import cv2

import sys

sys.path.append("/home/ofer/caffe/python")

import matplotlib.pyplot as plt

import matplotlib.cm as cm

import caffe

import lmdb

import warnings

warnings.filterwarnings("ignore", message="numpy.dtype size changed")

warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

import numpy as np

from PIL import Image

from config import cifar10_config as config

import scipy

from scipy.ndimage import gaussian_filter

from scipy.ndimage import rotate

import argparse

import math

from caffe.proto import caffe_pb2

caffe.set_mode_cpu()


net = caffe.Net('caffe/models/miniVggNet/m3/deploy2_3_miniVggNet.prototxt','caffe/models/miniVggNet/m3/snapshot_3_miniVggNet_/2-conv-layers/solver2_3_miniVggNet_iter_10000.caffemodel',caffe.TEST)


TEST_DATASET ="input/cifar10_jpg/val/*/*.jpg"


test_img_paths = [img_path for img_path in glob.glob(TEST_DATASET)]


NUMEL = len(test_img_paths)


print(NUMEL)


test_ids = np.zeros(([NUMEL,1]))


print(test_ids)


preds = np.zeros(([NUMEL, 3]))


idx = 0


tot_true  = 0


tot_false = 0


top5_true = 0

top5_false= 0


lmdb_file = "input/lmdb/train_lmdb"

lmdb_env = lmdb.open(lmdb_file)

lmdb_txn = lmdb_env.begin()

lmdb_cursor = lmdb_txn.cursor()

datum = caffe_pb2.Datum()

print(datum.channels,datum.height,datum.width)
k=0

print(lmdb_cursor)

for key, value in lmdb_cursor:

k+=1

datum.ParseFromString(value)

label = datum.label

data=np.fromstring(datum.data, dtype=np.float32).reshape(datum.channels, datum.height, datum.width)

img=data

img=np.rollaxis(img, 1, 0)

img=np.rollaxis(img, 2, 0)

img=np.rollaxis(img, 2, 0)

net.blobs['data'].data[...] = img

out = net.forward()

best_n = net.blobs['prob'].data[0].flatten().argsort()[-1: -6:-1]

print("DBG INFO: ", best_n)

pred_probas = out['prob']

print(pred_probas)

top5 = pred_probas.argsort()[-5:][::-1]

if 'good' in img_path:

label = 0

elif 'miss' in img_path:

label = 1

elif 'excess' in img_path:
label = 2  

else:

label = -1 # non existing

if label in top5 :

top5_true = top5_true + 1

else :

top5_false = top5_false + 1

test_ids[idx] = label
preds[idx] = pred_probas


print("IMAGE: " + img_path)

print("PREDICTED: %d" % preds[idx].argmax())

print("EXPECTED : %d" % test_ids[idx])

print '-------'

from sklearn.preprocessing import LabelBinarizer

from sklearn.metrics import classification_report

lb     = LabelBinarizer()

testY  = lb.fit_transform(test_ids)

labelNames = ["good", "miss", "excess"]

report=classification_report(testY.argmax(axis=1), preds.argmax(axis=1), target_names=labelNames)

print(report)

from sklearn.metrics import accuracy_score

print('SKLEARN Accuracy = %.2f' % accuracy_score(testY.argmax(axis=1), preds.argmax(axis=1)) )

list_predictions = np.array(preds.argmax(axis=1)) # actual predictions
list_str_num = np.array(testY.argmax(axis=1))    # ground truth

for ii in range(0, NUMEL) :
n1 = list_str_num[ii]
n2 = list_predictions[ii]
diff = n1 - n2
if diff == 0 :
tot_true = tot_true + 1
else:
tot_false = tot_false+1

top5_accuracy = float(top5_true) / (top5_true + top5_false)

print("\n")

print('TOP-5 ACCURACY                    = %.2f ' % top5_accuracy)

print 'TOP-5 FALSE                       = ', top5_false

print 'TOP-5 TRUE                        = ', top5_true

print("\n")

print 'TOTAL NUMBER OF TRUE  PREDICTIONS = ', tot_true

print 'TOTAL NUMBER OF FALSE PREDICTIONS = ', tot_false

if (tot_true+tot_false) != NUMEL :

print 'ERROR: number of total false and positive is not equal to the number of processed images'

if (top5_true+top5_false) != NUMEL :

print 'ERROR: number of top5 total false and positive is not equal to the number of processed images'        

recall =  float(tot_true)/(tot_true+tot_false)

print('MANUALLY COMPUTED RECALL = %.2f ' % recall)
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.