So, I am trying to implement a neural network in Python by only using NumPy. I have tried to do this by following 3Blue1Brown's video's about the topic, however, when testing my implementation, the network does not seem to work fully. I am testing the implementation on the AND, OR, and XOR problems - the AND and OR problems run as expected, however, the XOR problem does not:
Input Output:
0, 0 0.253
1, 0 0.793
0, 1 0.793
1, 1 0.793 <- WRONG!
These results have been generated with a sample size of 1000 samples chosen randomly, 10 epochs, and a neural network with one hidden layer of two nodes, and the sigmoid function in both the hidden layer and in the output layer. If I try a different model, for instance with two hidden layers with each three nodes, I get much worse results:
Input: Output:
0, 0 0.712 <- Wrong
1, 0 0.712
0, 1 0.712
1, 1 0.712 <- Wrong
However, for some reason a model with no hidden layers does not as bad:
Input: Output:
0, 0 0.025
1, 0 0.999
0, 1 0.984
1, 1 0.945 <- Still wrong tho
I have tried different amounts of testing data, epochs, dimensions in the hidden layers, and hidden layers, and nothing seems to work
So, my question is if anyone has any idea why i get the wrong result in the XOR problem but not in the AND and the OR problems? And if so, how to fix it?
My implementation is as follows:
import numpy as np
from HiddenLayer import HiddenLayer
from InputLayer import InputLayer
np.random.seed(1) # Picking seed for debugging
class NeuralNetwork:
def __init__(self, input_dimensions):
self.n_layers = 1
self.layers = np.array([InputLayer(input_dimensions)])
def add_layer(self, dimensions):
self.layers = np.append(self.layers, HiddenLayer(dimensions, self.layers[self.n_layers - 1].dimensions))
self.n_layers += 1
@staticmethod
def sigmoid_derived(x):
return np.multiply(x, 1.0 - x)
def train(self, X, t, n_epoch, learning_rate = 1.0):
def update_weight(l, j, k):
self.layers[l].weights[j, k] -= learning_rate * self.layers[l - 1].a[k] * NeuralNetwork.sigmoid_derived(self.layers[l].a[j]) * self.layers[l].grad[j]
def update_bias(l, j):
self.layers[l].b[j] -= learning_rate * NeuralNetwork.sigmoid_derived(self.layers[l].a[j]) * self.layers[l].grad[j]
for _ in range(n_epoch):
for x, _t in zip(X, t):
# Forwardpropagation
self._forwardprop(x)
# Backpropagation
for l in reversed(range(1, self.n_layers)): # Iterates through each layer (except input layer)
for j in range(self.layers[l].dimensions): # Iterates through each node of layer l
self.layers[l].grad[j] = 0
for k in range(self.layers[l - 1].dimensions):
if (l == self.n_layers - 1): # If layer l is the output layer
self.layers[l].grad[j] = 1/len(t[0]) * 2 * (self.layers[l].a[j] - _t[j])
else:
for _j in range(self.layers[l + 1].dimensions):
self.layers[l].grad[j] += self.layers[l + 1].weights[_j, k] * NeuralNetwork.sigmoid_derived(self.layers[l + 1].a[_j]) * self.layers[l + 1].grad[_j]
update_weight(l, j, k)
update_bias(l, j)
def _forwardprop(self, x):
self.layers[0].feedforward(x)
for l in range(1, self.n_layers):
self.layers[l].feedforward(self.layers[l].weights, self.layers[l - 1].a)
def predict(self, x):
self._forwardprop(x)
return self.layers[self.n_layers - 1].a
import numpy as np
class HiddenLayer:
def __init__(self, dimensions, dim_from):
self.dimensions = dimensions
self.a = np.zeros((dimensions, 1))
self.b = np.random.rand(dimensions, 1)
self.grad = np.zeros((dimensions, 1))
self.weights = np.random.rand(dimensions, dim_from)
def feedforward(self, input_weights, input_a):
for i in range(self.dimensions):
self.a[i] = 1/(1 + np.exp(-1 * (np.matmul(input_weights[i], input_a) + self.b[i])))
import numpy as np
class InputLayer:
def __init__(self, dimensions):
self.dimensions = dimensions
self.a = np.zeros((dimensions, 1))
def feedforward(self, x):
self.a = x
I know it is quit a bit of code - please let me know if there is anything you find ambiguous :)
