# How to compute Frechet Inception Score for MNIST GAN?

I'm starting out with GANs and I am training a DC-GAN on MNIST dataset. The two metrics that are used to evaluate GANs are Inception Score (IS) and Frechet Inception Distance (FID). Since Inception network is not trained to classify MNIST digits, I built a simple MNIST classifier and I'm using that. Inception score, I'm getting pretty decent values. For FID, I'm using the output of last Fully Connected Layer as feature vector. But, I'm getting values in the order of $$10^6$$, which doesn't look right. So, I have a few questions

1. Does it make sense to compute FID for MNIST GAN?
2. Is the value in the order of $$10^6$$ correct?
3. How many images from real dataset should be used while computing FID
4. Is my code below correct? I used the code provided by original authors and modified it according to my requirements.

If you can answer any of these questions, even partially, that would be of immense help to me. Thanks!

Code: FidScore.py

import warnings

import numpy
from keras.datasets import mnist
from keras.models import Model
from scipy import linalg

from DcGanBaseModel import DcGanBaseModel
from MnistClassifierModel06 import MnistClassifier06
from mnist.MnistModel02 import MnistModel02

class FrechetInceptionDistance:

def __init__(self, real_activations: numpy.ndarray, verbose=False) -> None:
self.real_activations = real_activations
self.verbose = verbose

self.real_mu = numpy.mean(real_activations, axis=0)
self.real_sigma = numpy.cov(real_activations, rowvar=False)

def compute_fid(self, fake_activations: numpy.ndarray):
fake_mu = numpy.mean(fake_activations, axis=0)
fake_sigma = numpy.cov(fake_activations, rowvar=False)
fid = self.calculate_frechet_distance(fake_mu, fake_sigma, self.real_mu, self.real_sigma)
return fid

@staticmethod
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):
"""
https://github.com/bioinf-jku/TTUR/blob/master/FIDvsINC/fid.py#L99-L148
Numpy implementation of the Frechet Distance.
The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
and X_2 ~ N(mu_2, C_2) is
d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).
Stable version by Dougal J. Sutherland.
Params:
-- mu1 : Numpy array containing the activations of the pool_3 layer of the
inception net ( like returned by the function 'get_predictions')
for generated samples.
-- mu2   : The sample mean over activations of the pool_3 layer, precalcualted
on an representive data set.
-- sigma1: The covariance matrix over activations of the pool_3 layer for
generated samples.
-- sigma2: The covariance matrix over activations of the pool_3 layer,
precalcualted on an representive data set.
Returns:
--   : The Frechet Distance.
"""

mu1 = numpy.atleast_1d(mu1)
mu2 = numpy.atleast_1d(mu2)

sigma1 = numpy.atleast_2d(sigma1)
sigma2 = numpy.atleast_2d(sigma2)

assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"

diff = mu1 - mu2

# product might be almost singular
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
if not numpy.isfinite(covmean).all():
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
offset = numpy.eye(sigma1.shape) * eps
covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))

# numerical error might give slight imaginary component
if numpy.iscomplexobj(covmean):
if not numpy.allclose(numpy.diagonal(covmean).imag, 0, atol=1e-3):
m = numpy.max(numpy.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real

tr_covmean = numpy.trace(covmean)

return diff.dot(diff) + numpy.trace(sigma1) + numpy.trace(sigma2) - 2 * tr_covmean

def compute_fid_score_for_gan(gan_model: DcGanBaseModel, classifier_model, layer_name, num_classes):
# Define Feature Extracter
feature_layer = Model(inputs=classifier_model.model.input,
outputs=classifier_model.model.get_layer(layer_name).output)

# Compute Features for MNIST Dataset Images
x_train = x_train.reshape(x_train.shape, x_train.shape, x_train.shape, 1)
real_features = feature_layer.predict(x_train)
fid = FrechetInceptionDistance(real_features, verbose=True)

num_images = num_classes * 1000
gen_images = gan_model.generate_images(num_images)
fake_features = feature_layer.predict(gen_images)

fid_score = fid.compute_fid(fake_features)
return fid_score

def demo1():
gan_model = MnistModel02(print_model_summary=False)
'../../../../../DiscriminativeModels/01_MNIST_Classification/Runs/Run01/Trained_Models/MNIST_Model_10.h5')
fid_score = compute_fid_score_for_gan(gan_model, classifier_model, 'dense_1', 10)
print(fid_score)

if __name__ == '__main__':
demo1()


MnistClassifierModel06.py

import math
import numpy
from keras import Sequential
from keras.layers import Convolution2D, Dense, Dropout, Flatten, MaxPooling2D

from MnistClassifierModelBase import MnistClassifier

class MnistClassifier06(MnistClassifier):
def __init__(self, verbose: bool = False):
super().__init__(verbose)
self.model = None
self.verbose = verbose

def build_model(self):
self.model = Sequential()
self.model.add(Convolution2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))

self.verbose_log(self.model.summary())
self.compile_model()

def compile_model(self):

def fit(self, x, y, batch_size=32, epochs=30):
history = self.model.fit(x, y, batch_size, epochs)
return history.history['loss'], history.history['acc']

def train_on_batch(self, x: numpy.ndarray, y: numpy.ndarray, batch_size=32):
num_samples = x.shape
if num_samples % batch_size != 0:
raise RuntimeWarning('Batch size does not divide number of samples exactly. '
'Last set of samples will not be used for training')
loss, accuracy = [], []
self.verbose_log('iteration, loss, accuracy')
for i in range(int(math.floor(num_samples / batch_size))):
data = x[i * batch_size:(i + 1) * batch_size, :, :]
labels = y[i * batch_size:(i + 1) * batch_size]
batch_loss, batch_accuracy = self.model.train_on_batch(data, labels)
loss.append(batch_loss)
accuracy.append(batch_accuracy)
self.verbose_log('{0:04},{1:2.4f},{2:0.4f}\n'.format(i + 1, batch_loss, batch_accuracy))
return loss, accuracy

def evaluate(self, x, y):
score = self.model.evaluate(x, y)
return score, score

def classify(self, x):
y = self.model.predict(x)
return y

def save_model(self, save_path):
self.model.save(save_path)

def save_model_data(self, json_path, weights_path):
with open(json_path, 'w') as json_file:
json_file.write(self.model.to_json())
self.model.save_weights(weights_path)

@staticmethod
classifier = MnistClassifier06()
classifier.model = model
return classifier

@staticmethod