1
$\begingroup$

I am starting with deep learning and decided to code a backpropagation algorithm on Python 3. I have followed many tutorials and have taken as example many programs that work. Yet, for some reason, my supervised Neural Network can't seem to solve an XOR gate after 1000000 epochs. Can someone please find my mistake?

import random
import math
import numpy as np

def sigmoid(x):
    if x < -100:
        return 0
    if x > 100:
        return 1
    return 1 / (1 + np.exp(-x))

def derivative_sigmoid_value(value):
    return value * (1 - value)

def derivative_sigmoid_z(z):
    return np.exp(-z) / ((1 + np.exp(-z))**2)

class Weight:
    def __init__(self):

        #Initialise randomly
        self.weight = random.uniform(-1, 1)

        #Variables that will store the sum of the wished changes for each example and then do the average later
        self.total_wished_changes = 0
        self.number_of_wished_changes = 0


class Neuron:
    def __init__(self, layer, number, num_output_weights=0):
        #Classifying variables
        self.num_output_weights = num_output_weights
        self.layer = layer
        self.number = number

        #Value of the neuron before the non-linear function
        self.z = 0
        #Value of the neuron after the non-linear function
        self.value = 0

        #Initialise the weights coming from this neuron
        self.output_weights = []
        for i in range(self.num_output_weights):
            self.output_weights.append(Weight())

        #Value of the bias
        self.bias_weight = 0

        # Variables that will store the sum of the wished changes for the bias for each example and then do the average later
        self.total_wished_bias_changes = 0
        self.number_wished_bias_changes = 0

        #Calculated value of the derivative of the neuron with respect to the cost, times the derivative of the sigmoid
        self.delta = 0





class Net:
    def __init__(self, topology, learning_rate=0.5, bias=1):
        #Number of neurons for each layer
        self.topology = topology

        #Number of layers
        self.num_layers = len(self.topology)

        #Value that will multiply the bias_weight value of each neuron, 1 by default
        self.bias = bias

        #Values to keep track of training from outside
        self.expected = None
        self.result = None
        self.error = 0

        #Self.layers is where all the neurons are stored. This is just to initialise it
        self.topology.append(0)
        self.layers = []
        for layer in range(self.num_layers):
            #Create a layer, add it a number of neurons and append it to self.layers
            layer_toAdd = []
            for i in range(self.topology[layer]):
                layer_toAdd.append(Neuron(layer, i, self.topology[layer + 1]))
            self.layers.append(layer_toAdd)
        self.topology.pop()

        self.learning_rate = learning_rate

    #Function that goes through each of the neurons of a certain layer and calculates z, then its actual value
    def calculate_layer(self, layer):
        for neuron_idx in range(self.topology[layer]):
            neuron = self.layers[layer][neuron_idx]
            #Z= sum of all the previous layers values * the weights + the bias
            neuron.z = sum(
                [inputNeuron.value * inputNeuron.output_weights[neuron_idx].weight for inputNeuron in
                 self.layers[layer - 1]]) + self.bias * neuron.bias_weight

            #Apply the sigmoid function to z to get the actual value
            neuron.value = sigmoid(neuron.z)

    #Function that initialises the first layer acording to the inputs, then calculates each layer according to its previous one
    def calculateResult(self, inputs):
        #Check for the correct number of inputs
        if len(inputs) != self.topology[0]:
            raise ValueError("Number of inputs must be equal to the number of input neurons: "
                             "expected {0} and got {1}".format(self.topology[0],len(inputs)))

        #Input layer
        for i in range(len(inputs)):
            self.layers[0][i].value = inputs[i]

        #Calculate values of each layer
        for layer in range(1, self.num_layers):
            self.calculate_layer(layer)

        #return the values of the output layer
        return [neuron.value for neuron in self.layers[-1]]

    #Function that takes inputs and expected outputs and calculates the wished changes for each bias and weight
    def trainOnOneExample(self, inputs, expected_outputs):
        #Get the nets prediction
        outputs_got = self.calculateResult(inputs)

        #Calculate prediction's total cost
        self.error = sum([(expected_outputs[i] - outputs_got[i]) ** 2 for i in range(len(expected_outputs))])
        self.expected = expected_outputs
        self.result = outputs_got
        
        #   As the Derivative of a neuron value is always multiplied by the derivative of the sigmoid, I call this product delta and use that value
        #A: ∂neuron_value/∂Cost: Σ(per weight): ((weight value) x (σʹ(z)) x (∂neuron_value(il leads to)/∂Cost)).
        
        #B: ∂weight/∂Cost: (value of the neuron it comes from) x (σʹ(z)) x (∂neuron_value(it leads to)/∂Cost) = neuron_delta
        #C: Derivative of the bias: (σʹ(z)) x (∂neuron_value/∂Cost) = neuron_delta

        for layer_idx in range(self.num_layers-1,0,-1):
            for neuron_idx in range(self.topology[layer_idx]):
                neuron = self.layers[layer_idx][neuron_idx]
                #A
                #For the first layer, we use the derivative with the cost function
                if layer_idx == self.num_layers-1:
                    neuron_derVsCost = 2 * (neuron.value - expected_outputs[neuron_idx])
                    neuron.delta = neuron_derVsCost * derivative_sigmoid_z(neuron.z)
                else:
                    neuron_derVsCost = sum(
                        [neuron.output_weights[i].weight * self.layers[layer_idx + 1][i].delta
                         for i in range(self.topology[layer_idx + 1])])
                    neuron.delta = neuron_derVsCost * derivative_sigmoid_z(neuron.z)

                #C
                neuron.total_wished_bias_changes += neuron.delta
                neuron.number_wished_bias_changes += 1

                #Go throught each weight
                for weight in range(len(neuron.output_weights)):
                    this_weight = neuron.output_weights[weight]
                    #B
                    this_weight.total_wished_changes += neuron.value * self.layers[layer_idx+1][weight].delta
                    this_weight.number_of_wished_changes += 1


    def CorrectFromWished(self):
        #Iterate through all weights and biases and correct their value, using the average derivatives and multiplying it by the learning rate.
        for layer_idx in range(self.num_layers):
            if layer_idx == 0:
                continue
            layer = self.layers[layer_idx]
            for neuron in layer:
                neuron.bias_weight -= (neuron.total_wished_bias_changes / neuron.number_wished_bias_changes) * self.learning_rate
                #print(neuron.number_wished_bias_changes)
                neuron.total_wished_bias_changes = 0
                neuron.number_wished_bias_changes = 0

                for weight in neuron.output_weights:
                    weight.weight -= (weight.total_wished_changes / weight.number_of_wished_changes) * self.learning_rate
                    weight.total_wished_changes = 0
                    weight.number_of_wished_changes = 0

    def simple_training(self, training_data, num_iterations=100000):
        #A funtion that goes through random pieces of the training data, trains on it and instantly corrects.
        
        #Variables to make an average of the last X percent: the sum of all and the value of X
        last_errors = 0
        last_percentage = 1
        for i in range(num_iterations):
            sample = random.choice(training_data)
            expectedoutput, inputs  = sample

            self.trainOnOneExample(inputs, expectedoutput)

            if i > num_iterations*(1-last_percentage/100):
                last_errors += self.error

            print("Epoch",i,"Expected",self.expected,"and got",self.result,"-> total error of",self.error)
            self.CorrectFromWished()

        print("Average of errors of last",str(last_percentage)+"%:",100*last_errors/(num_iterations*last_percentage))


net = Net([2, 2, 1])

test_data=[[[0],[1,1]],[[1],[0,1]],[[1],[1,0]],[[0],[0,0]]]
training_data=test_data*30

net.simple_training(training_data, 1000000)
```
$\endgroup$

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.