# Deep learning through backpropagation: not learning

I am starting with deep learning and decided to code a backpropagation algorithm on Python 3. I have followed many tutorials and have taken as example many programs that work. Yet, for some reason, my supervised Neural Network can't seem to solve an XOR gate after 1000000 epochs. Can someone please find my mistake?

import random
import math
import numpy as np

def sigmoid(x):
if x < -100:
return 0
if x > 100:
return 1
return 1 / (1 + np.exp(-x))

def derivative_sigmoid_value(value):
return value * (1 - value)

def derivative_sigmoid_z(z):
return np.exp(-z) / ((1 + np.exp(-z))**2)

class Weight:
def __init__(self):

#Initialise randomly
self.weight = random.uniform(-1, 1)

#Variables that will store the sum of the wished changes for each example and then do the average later
self.total_wished_changes = 0
self.number_of_wished_changes = 0

class Neuron:
def __init__(self, layer, number, num_output_weights=0):
#Classifying variables
self.num_output_weights = num_output_weights
self.layer = layer
self.number = number

#Value of the neuron before the non-linear function
self.z = 0
#Value of the neuron after the non-linear function
self.value = 0

#Initialise the weights coming from this neuron
self.output_weights = []
for i in range(self.num_output_weights):
self.output_weights.append(Weight())

#Value of the bias
self.bias_weight = 0

# Variables that will store the sum of the wished changes for the bias for each example and then do the average later
self.total_wished_bias_changes = 0
self.number_wished_bias_changes = 0

#Calculated value of the derivative of the neuron with respect to the cost, times the derivative of the sigmoid
self.delta = 0

class Net:
def __init__(self, topology, learning_rate=0.5, bias=1):
#Number of neurons for each layer
self.topology = topology

#Number of layers
self.num_layers = len(self.topology)

#Value that will multiply the bias_weight value of each neuron, 1 by default
self.bias = bias

#Values to keep track of training from outside
self.expected = None
self.result = None
self.error = 0

#Self.layers is where all the neurons are stored. This is just to initialise it
self.topology.append(0)
self.layers = []
for layer in range(self.num_layers):
#Create a layer, add it a number of neurons and append it to self.layers
for i in range(self.topology[layer]):
self.topology.pop()

self.learning_rate = learning_rate

#Function that goes through each of the neurons of a certain layer and calculates z, then its actual value
def calculate_layer(self, layer):
for neuron_idx in range(self.topology[layer]):
neuron = self.layers[layer][neuron_idx]
#Z= sum of all the previous layers values * the weights + the bias
neuron.z = sum(
[inputNeuron.value * inputNeuron.output_weights[neuron_idx].weight for inputNeuron in
self.layers[layer - 1]]) + self.bias * neuron.bias_weight

#Apply the sigmoid function to z to get the actual value
neuron.value = sigmoid(neuron.z)

#Function that initialises the first layer acording to the inputs, then calculates each layer according to its previous one
def calculateResult(self, inputs):
#Check for the correct number of inputs
if len(inputs) != self.topology[0]:
raise ValueError("Number of inputs must be equal to the number of input neurons: "
"expected {0} and got {1}".format(self.topology[0],len(inputs)))

#Input layer
for i in range(len(inputs)):
self.layers[0][i].value = inputs[i]

#Calculate values of each layer
for layer in range(1, self.num_layers):
self.calculate_layer(layer)

#return the values of the output layer
return [neuron.value for neuron in self.layers[-1]]

#Function that takes inputs and expected outputs and calculates the wished changes for each bias and weight
def trainOnOneExample(self, inputs, expected_outputs):
#Get the nets prediction
outputs_got = self.calculateResult(inputs)

#Calculate prediction's total cost
self.error = sum([(expected_outputs[i] - outputs_got[i]) ** 2 for i in range(len(expected_outputs))])
self.expected = expected_outputs
self.result = outputs_got

#   As the Derivative of a neuron value is always multiplied by the derivative of the sigmoid, I call this product delta and use that value
#A: ∂neuron_value/∂Cost: Σ(per weight): ((weight value) x (σʹ(z)) x (∂neuron_value(il leads to)/∂Cost)).

#B: ∂weight/∂Cost: (value of the neuron it comes from) x (σʹ(z)) x (∂neuron_value(it leads to)/∂Cost) = neuron_delta
#C: Derivative of the bias: (σʹ(z)) x (∂neuron_value/∂Cost) = neuron_delta

for layer_idx in range(self.num_layers-1,0,-1):
for neuron_idx in range(self.topology[layer_idx]):
neuron = self.layers[layer_idx][neuron_idx]
#A
#For the first layer, we use the derivative with the cost function
if layer_idx == self.num_layers-1:
neuron_derVsCost = 2 * (neuron.value - expected_outputs[neuron_idx])
neuron.delta = neuron_derVsCost * derivative_sigmoid_z(neuron.z)
else:
neuron_derVsCost = sum(
[neuron.output_weights[i].weight * self.layers[layer_idx + 1][i].delta
for i in range(self.topology[layer_idx + 1])])
neuron.delta = neuron_derVsCost * derivative_sigmoid_z(neuron.z)

#C
neuron.total_wished_bias_changes += neuron.delta
neuron.number_wished_bias_changes += 1

#Go throught each weight
for weight in range(len(neuron.output_weights)):
this_weight = neuron.output_weights[weight]
#B
this_weight.total_wished_changes += neuron.value * self.layers[layer_idx+1][weight].delta
this_weight.number_of_wished_changes += 1

def CorrectFromWished(self):
#Iterate through all weights and biases and correct their value, using the average derivatives and multiplying it by the learning rate.
for layer_idx in range(self.num_layers):
if layer_idx == 0:
continue
layer = self.layers[layer_idx]
for neuron in layer:
neuron.bias_weight -= (neuron.total_wished_bias_changes / neuron.number_wished_bias_changes) * self.learning_rate
#print(neuron.number_wished_bias_changes)
neuron.total_wished_bias_changes = 0
neuron.number_wished_bias_changes = 0

for weight in neuron.output_weights:
weight.weight -= (weight.total_wished_changes / weight.number_of_wished_changes) * self.learning_rate
weight.total_wished_changes = 0
weight.number_of_wished_changes = 0

def simple_training(self, training_data, num_iterations=100000):
#A funtion that goes through random pieces of the training data, trains on it and instantly corrects.

#Variables to make an average of the last X percent: the sum of all and the value of X
last_errors = 0
last_percentage = 1
for i in range(num_iterations):
sample = random.choice(training_data)
expectedoutput, inputs  = sample

self.trainOnOneExample(inputs, expectedoutput)

if i > num_iterations*(1-last_percentage/100):
last_errors += self.error

print("Epoch",i,"Expected",self.expected,"and got",self.result,"-> total error of",self.error)
self.CorrectFromWished()

print("Average of errors of last",str(last_percentage)+"%:",100*last_errors/(num_iterations*last_percentage))

net = Net([2, 2, 1])

test_data=[[[0],[1,1]],[[1],[0,1]],[[1],[1,0]],[[0],[0,0]]]
training_data=test_data*30

net.simple_training(training_data, 1000000)
$$$$
`