As the title says, i made an RBM from scratch:
class RBM:
def __init__(self, data, n_v, n_h, k, epochs, mini_batch_size, alpha, momentum, weight_decay):
self.number_features = 10
self.n_v = n_v
self.n_h = self.number_features
self.k = k
self.alpha = alpha
self.momentum = momentum
self.weight_decay = weight_decay
self.mini_batch_size = mini_batch_size
self.epochs = epochs
self.data = data
def sigmoid(self, x):
x = np.clip(x, -25, 25)
return 1 / (1 + np.exp(-x))
def fit(self):
np.random.seed(100)
self.data = self.data / 255.0
self.n_v = len(self.data[0])
self.n_h = self.number_features
self.w = np.random.randn(self.n_v, self.n_h) * 0.1 # initialization of weights, a matrix of shape (v1, h0)
self.a = np.ones(self.n_v) * 0.5 # initialization of the biases in the forward step, matrix of size v1
self.b = np.ones(self.n_h) # initialization of the biases in the reconstruction step, matrix of size h1 (which is equal to h0)
self.w_moment = np.zeros((self.n_v, self.n_h))
self.a_moment = np.zeros((self.n_v))
self.b_moment = np.zeros((self.n_h))
self.train()
def train(self):
for epoch_no in range(self.epochs): # iterates through epochs
if epoch_no == 100: self.momentum = 0.9
ep_error = 0 # initialization of the error for each epoch
for i in range(0, len(self.data), self.mini_batch_size): # data is splitted into batches
mini_batch = self.data[i:i+self.mini_batch_size]
ep_error += self.contrastive_divergence(mini_batch) # contrastive divergence is added to error of epoch ep
print("Epoch Number: ", epoch_no, "Error: ", ep_error.item())
def contrastive_divergence(self, v):
p_h_0 = self.sample_hidden(v)
h_0 = (p_h_0 >= np.random.rand(self.n_h))
g_0 = np.dot(v.T, h_0)
wv_a = h_0
for step in range(self.k):
p_v_h = self.sample_visible(wv_a)
p_h_v = self.sample_hidden(p_v_h)
wv_a = (p_h_v >= np.random.rand(self.n_h))
p_v_k = p_v_h
p_h_k = p_h_v
g_k = np.dot(p_v_k.T, p_h_k)
self.update_parameters(g_0, g_k, v, p_v_k, p_h_0, p_h_k)
error = np.sum((v - p_v_k) ** 2) / len(p_v_k)
return error
def sample_hidden(self, p_v_h):
wv = np.dot(p_v_h, self.w)
wv_a = wv + self.b
p_h_v = self.sigmoid(wv_a)
return p_h_v
def sample_visible(self, p_h_v):
wh = np.dot(p_h_v, self.w.T)
wh_b = wh + self.a
p_v_h = self.sigmoid(wh_b)
return p_v_h
def update_parameters(self, g_0, g_k, v, p_v_k, p_h_0, p_h_k):
self.w_moment *= self.momentum
del_w = (g_0 - g_k) + self.w_moment
self.a_moment *= self.momentum
del_a = np.sum(v - p_v_k, axis=0) + self.a_moment
self.b_moment *= self.momentum
del_b = np.sum(p_h_0 - p_h_k, axis=0) + self.b_moment
batch_size = v.shape[0]
self.w += (del_w * self.alpha / batch_size) - self.w * self.weight_decay
self.a += del_a * self.alpha / batch_size
self.b += del_b * self.alpha / batch_size
self.w_moment = del_w
self.a_moment = del_a
self.b_moment = del_b
def predict(self, x_test):
return self.sample_hidden(x_test)
And then i fit my model on MNIST:
from keras.datasets import mnist
(xt, yt), (xtt, ytt) = mnist.load_data()
xt = np.array([i.flatten() for i in xt])
rbm = RBM(xt[:2500], 0, 0, 2, 500, 64, 0.15, 0.5, 0.0001)
rbm.fit()
The error drops up to converge (not to a value next to 0), and when i try to predict an image this is one of the outputs:
Can someone tell me how to fix this? it may be in hyperparameters as well as in code
