I've been trying to implement the backpropagation algorithm using only numpy, I've already done the Keras version, but when implementing the numpy version, the loss is diverging as seen in the image below:

enter image description here

I initialized the weights using the glorot initialization (Keras default), used SGD with the same learning rate as in Keras, using sigmoid as activation function, but still, the algorithm can't converge for a simple XOR.

I think there is something wrong with the backward pass but I can't figure out what it is.

import numpy as np
import matplotlib.pyplot as plt

class MLP:
    def glorot_initializer(fan_in, fan_out):
        factor = 4 * np.sqrt(6 / (fan_in + fan_out))
        return np.random.uniform(-factor, factor, (fan_in, fan_out))

    def relu(x):
        return np.maximum(x, 0)

    def step(x):
        return np.heaviside(x, 1)

    def sigmoid(x):
        return 1 / (1 + np.exp(-x))

    def sigmoid_diff(x):
        sig = MLP.sigmoid(x)
        return sig * (1 - sig)

    def __init__(self, input_size, output_size, hidden_layers, hidden_size):
        self.input_size = input_size
        self.output_size = output_size
        self.hidden_size = hidden_size
        self.layers = list()
        self.grads = list()

        # input layer
        W = self.glorot_initializer(input_size, hidden_size)
        b = np.zeros((hidden_size, 1))
        self.layers.append(dict(W=W, b=b, type='input'))

        # hidden layers
        for i in range(hidden_layers):
            W = self.glorot_initializer(hidden_size, hidden_size)
            b = np.zeros((hidden_size, 1))
            layer = dict(W=W, b=b, type='hidden')

        # output layer
        W = self.glorot_initializer(hidden_size, output_size)
        b = np.zeros((output_size, 1))
        self.layers.append(dict(W=W, b=b, type='output'))

    def forward_pass(self, x):
        for layer in self.layers:
            x = np.dot(x, layer['W']) + layer['b'].T
            layer['output'] = x
            x = self.sigmoid(x)
            layer['activation_output'] = x
        return x

    def backward_pass(self, x, y, forward_pass):
        m = y.shape[0]
        dlda = 2*(y - forward_pass)  # considering MSE
        dlda = np.sum(dlda, axis=0, keepdims=True) / m
        self.grads = list()

        for i in range(len(self.layers)-1, -1, -1):
            grad = dict()

            z = self.layers[i]['output']
            dadz = np.sum(self.sigmoid_diff(z), axis=0, keepdims=True) / m
            if self.layers[i]['type'] == 'input':
                dzdw = np.sum(x, axis=0, keepdims=True) / m
                dzdw = np.sum(self.layers[i-1]['activation_output'], axis=0, keepdims=True) / m

            grad['b'] = dlda.dot(dadz).T
            grad['W'] = dlda.dot(dadz).T.dot(dzdw).T
            dzda_prev = self.layers[i]['W'].T
            dlda = np.sum(dlda.dot(dadz).dot(dzda_prev), axis=1, keepdims=True)


        return self.grads

    def update_parameters(self, lr=0.2):
        for layer, grad in zip(self.layers, self.grads):
            layer['W'] -= lr * grad['W']
            layer['b'] -= lr * grad['b']

if __name__ == '__main__':
    epochs = 20000
    input = np.array([[0, 0], [1, 1], [0, 1], [1, 0]])
    target = np.array([[0.0], [0.0], [1.0], [1.0]])

    input_size = input.shape[1]
    output_size = target.shape[1]
    mlp = MLP(input_size=input_size, output_size=output_size, hidden_layers=2, hidden_size=2)

    loss = np.zeros(epochs)

    for i in range(epochs):
        fp = mlp.forward_pass(input)
        bp = mlp.backward_pass(input, target, fp)
        loss[i] = np.sum((fp-target)**2/fp.shape[0])


    plt.title('LOSS x EPOCHS')

Any help, would be very much appreciated!

  • $\begingroup$ Lucas, it would be nice if you refer to some guys repository who implemented it.. like I did it for MNIST.. $\endgroup$
    – Aditya
    Jan 6 '19 at 2:21

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.