I'm starting out with GANs and I am training a DC-GAN on MNIST dataset. The two metrics that are used to evaluate GANs are Inception Score (IS) and Frechet Inception Distance (FID). Since Inception network is not trained to classify MNIST digits, I built a simple MNIST classifier and I'm using that. Inception score, I'm getting pretty decent values. For FID, I'm using the output of last Fully Connected Layer as feature vector. But, I'm getting values in the order of $10^6$, which doesn't look right. So, I have a few questions
- Does it make sense to compute FID for MNIST GAN?
- Is the value in the order of $10^6$ correct?
- How many images from real dataset should be used while computing FID
- Is my code below correct? I used the code provided by original authors and modified it according to my requirements.
If you can answer any of these questions, even partially, that would be of immense help to me. Thanks!
Code: FidScore.py
import warnings
import numpy
from keras.datasets import mnist
from keras.models import Model
from scipy import linalg
from DcGanBaseModel import DcGanBaseModel
from MnistClassifierModel06 import MnistClassifier06
from mnist.MnistModel02 import MnistModel02
class FrechetInceptionDistance:
def __init__(self, real_activations: numpy.ndarray, verbose=False) -> None:
self.real_activations = real_activations
self.verbose = verbose
self.real_mu = numpy.mean(real_activations, axis=0)
self.real_sigma = numpy.cov(real_activations, rowvar=False)
def compute_fid(self, fake_activations: numpy.ndarray):
fake_mu = numpy.mean(fake_activations, axis=0)
fake_sigma = numpy.cov(fake_activations, rowvar=False)
fid = self.calculate_frechet_distance(fake_mu, fake_sigma, self.real_mu, self.real_sigma)
return fid
@staticmethod
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):
"""
https://github.com/bioinf-jku/TTUR/blob/master/FIDvsINC/fid.py#L99-L148
Numpy implementation of the Frechet Distance.
The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
and X_2 ~ N(mu_2, C_2) is
d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).
Stable version by Dougal J. Sutherland.
Params:
-- mu1 : Numpy array containing the activations of the pool_3 layer of the
inception net ( like returned by the function 'get_predictions')
for generated samples.
-- mu2 : The sample mean over activations of the pool_3 layer, precalcualted
on an representive data set.
-- sigma1: The covariance matrix over activations of the pool_3 layer for
generated samples.
-- sigma2: The covariance matrix over activations of the pool_3 layer,
precalcualted on an representive data set.
Returns:
-- : The Frechet Distance.
"""
mu1 = numpy.atleast_1d(mu1)
mu2 = numpy.atleast_1d(mu2)
sigma1 = numpy.atleast_2d(sigma1)
sigma2 = numpy.atleast_2d(sigma2)
assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
# product might be almost singular
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
if not numpy.isfinite(covmean).all():
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
offset = numpy.eye(sigma1.shape[0]) * eps
covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))
# numerical error might give slight imaginary component
if numpy.iscomplexobj(covmean):
if not numpy.allclose(numpy.diagonal(covmean).imag, 0, atol=1e-3):
m = numpy.max(numpy.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
tr_covmean = numpy.trace(covmean)
return diff.dot(diff) + numpy.trace(sigma1) + numpy.trace(sigma2) - 2 * tr_covmean
def compute_fid_score_for_gan(gan_model: DcGanBaseModel, classifier_model, layer_name, num_classes):
# Define Feature Extracter
feature_layer = Model(inputs=classifier_model.model.input,
outputs=classifier_model.model.get_layer(layer_name).output)
# Compute Features for MNIST Dataset Images
(x_train, _), _ = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], x_train.shape[2], 1)
real_features = feature_layer.predict(x_train)
fid = FrechetInceptionDistance(real_features, verbose=True)
num_images = num_classes * 1000
gen_images = gan_model.generate_images(num_images)
fake_features = feature_layer.predict(gen_images)
fid_score = fid.compute_fid(fake_features)
return fid_score
def demo1():
gan_model = MnistModel02(print_model_summary=False)
gan_model.load_generator_model('../../Runs/01_MNIST/Model02/Run01/TrainedModels/generator_model_10000.h5')
classifier_model = MnistClassifier06().load_model(
'../../../../../DiscriminativeModels/01_MNIST_Classification/Runs/Run01/Trained_Models/MNIST_Model_10.h5')
fid_score = compute_fid_score_for_gan(gan_model, classifier_model, 'dense_1', 10)
print(fid_score)
if __name__ == '__main__':
demo1()
MnistClassifierModel06.py
import math
import numpy
from keras import Sequential
from keras.engine.saving import load_model, model_from_json
from keras.layers import Convolution2D, Dense, Dropout, Flatten, MaxPooling2D
from MnistClassifierModelBase import MnistClassifier
class MnistClassifier06(MnistClassifier):
def __init__(self, verbose: bool = False):
super().__init__(verbose)
self.model = None
self.verbose = verbose
def build_model(self):
self.model = Sequential()
self.model.add(Convolution2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
self.model.add(Convolution2D(32, (3, 3), activation='relu'))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
self.model.add(Dropout(0.25))
self.model.add(Flatten())
self.model.add(Dense(128, activation='relu'))
self.model.add(Dropout(0.5))
self.model.add(Dense(10, activation='softmax'))
self.verbose_log(self.model.summary())
self.compile_model()
def compile_model(self):
self.model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
def fit(self, x, y, batch_size=32, epochs=30):
history = self.model.fit(x, y, batch_size, epochs)
return history.history['loss'], history.history['acc']
def train_on_batch(self, x: numpy.ndarray, y: numpy.ndarray, batch_size=32):
num_samples = x.shape[0]
if num_samples % batch_size != 0:
raise RuntimeWarning('Batch size does not divide number of samples exactly. '
'Last set of samples will not be used for training')
loss, accuracy = [], []
self.verbose_log('iteration, loss, accuracy')
for i in range(int(math.floor(num_samples / batch_size))):
data = x[i * batch_size:(i + 1) * batch_size, :, :]
labels = y[i * batch_size:(i + 1) * batch_size]
batch_loss, batch_accuracy = self.model.train_on_batch(data, labels)
loss.append(batch_loss)
accuracy.append(batch_accuracy)
self.verbose_log('{0:04},{1:2.4f},{2:0.4f}\n'.format(i + 1, batch_loss, batch_accuracy))
return loss, accuracy
def evaluate(self, x, y):
score = self.model.evaluate(x, y)
return score[0], score[1]
def classify(self, x):
y = self.model.predict(x)
return y
def save_model(self, save_path):
self.model.save(save_path)
def save_model_data(self, json_path, weights_path):
with open(json_path, 'w') as json_file:
json_file.write(self.model.to_json())
self.model.save_weights(weights_path)
@staticmethod
def load_model(model_path) -> MnistClassifier:
model = load_model(model_path)
classifier = MnistClassifier06()
classifier.model = model
return classifier
@staticmethod
def load_model_from_data(json_path, weights_path) -> MnistClassifier:
with open(json_path, 'r') as json_file:
model = model_from_json(json_file.read())
model.load_weights(weights_path)
classifier = MnistClassifier06()
classifier.model = model
classifier.compile_model()
return classifier
