I created an ANN in Python 3. My backpropagation algorithm seems to work up to a point where the gradient becomes very small. I am familiar with the vanishing gradient problem, but I found that it only applies to really deep network; my simple test network is no such network. It consists of an input layer (1 input node and bias), no hidden layers, and an output layer (1 output node). How do I stop the gradient from vanishing? Here is the code:
import numpy as np
from random import random
class Neural_Network(object):
def __init__(self):
# Create a simple deterministic network for testing
# Define Hyperparameters
self.inputLayerSize = 1
self.outputLayerSize = 1
self.hiddenLayerSize = 0
self.numHiddenLayer = 0
self.numExamples = 20
self.learningRate = 0.07 # LEARNING RATE
self.weightDecay = 0
# in -> out
self.weights = [] # stores matrices of each layer of weights
self.z = [] # stores matrices of each layer of weighted sums
self.a = [] # stores matrices of each layer of activity
self.biases = [] # stores all biases
self.biasNodes = []
# Biases are matrices that are added to activity matrix
# Dimensions -> numExamples_*hiddenLayerSize or numExamples_*outputLayerSize
# Biases for output layer
b = [0.5 for x in range(self.outputLayerSize)]
B = [b for x in range(self.numExamples)];
self.biases.append(np.mat(B))
# Bias nodes
b= [1 for x in range(self.numExamples)]
for i in range(self.numHiddenLayer+1):
self.biasNodes.append(np.mat(b).reshape([self.numExamples,1]))
# Weights (Parameters)
# Weight matrix between input and output layer
W = np.matrix("0.5");
self.weights.append(W)
def setBatchSize(self, numExamples):
# Changes the number of rows (examples) for biases
if (self.numExamples > numExamples):
self.biases = [b[:numExamples] for b in self.biases]
def hypTan(self, z):
# Apply hyperbolic tangent function
return (np.exp(z) - np.exp(-z)) / (np.exp(z) + np.exp(-z))
def hypTanPrime(self, z):
# Apply derivative hyperbolic tangent function
return 4/np.multiply((np.exp(z) + np.exp(-z)), (np.exp(z) + np.exp(-z)))
def forward(self, X):
# Propagate outputs through network
self.z = []
self.a = []
self.z.append(np.dot(X, self.weights[0]) + self.biases[0])
self.a.append(self.hypTan(self.z[0]))
yHat = self.a[-1]
return yHat
def backProp(self, X, y):
# Compute derivative wrt W
# out -> in
dJdWb = [] # stores matrices of each dJdWb value
dJdW = [] # stores matrices of each dJdW (equal in size to self.weights[])
delta = [] # stores matrices of each backpropagating error
result = () # stores dJdW and dJdWb
self.yHat = self.forward(X)
# Quantifying Error
print(np.linalg.norm(y-self.yHat)/np.linalg.norm(y+self.yHat))
delta.insert(0,np.multiply(-(y-self.yHat), self.hypTanPrime(self.z[-1]))) # delta = (y-yHat)(sigmoidPrime(final layer unactivated))
dJdW.insert(0, np.dot(X.T, delta[0]) + (self.weightDecay*self.weights[-1]))
dJdWb.insert(0, np.dot(self.biasNodes[-1].T, delta[0]) + (self.weightDecay*self.biases[-1])) # you need to backpropagate to bias nodes
result = (dJdW, dJdWb)
return result
def train(self, X, y):
for t in range(10000):
dJ = self.backProp(X, y)
dJdW = dJ[0]
dJdWb = dJ[1]
for i in range(len(dJdW)):
print("dJdW:", dJdW[i], sep = " ", end = "\n")
print("dJdWb:", dJdWb[i], sep = " ", end = "\n\n")
#print("Weights:", self.weights[i]);
self.weights[i] -= self.learningRate*dJdW[i]
self.biases[i] -= self.learningRate*dJdWb[i]
# Instantiating Neural Network
# Instantiating Neural Network
NN = Neural_Network() # create a deterministic NN for testing
x = np.matrix("0.025; 0.05; 0.075; 0.1; 0.125; 0.15; 0.175; 0.2; 0.225; 0.25; 0.275; 0.3; 0.325; 0.35; 0.375; 0.4; 0.425; 0.45; 0.475; 0.5")
y = np.matrix("0.05; 0.1; 0.15; 0.2; 0.25; 0.3; 0.35; 0.4; 0.45; 0.5; 0.55; 0.6; 0.65; 0.7; 0.75; 0.8; 0.85; 0.9; 0.95; 1.0")
# Training
print("INPUT: ", end = '\n')
print(x, end = '\n\n')
print("BEFORE TRAINING", NN.forward(x), sep = '\n', end = '\n\n')
print("ERROR: ")
NN.train(x,y)
print("\nAFTER TRAINING", NN.forward(x), sep = '\n', end = '\n\n')
NN.setBatchSize(1) # changing settings to receive one input at a time
while True:
inputs = input()
x = np.mat([float(i) for i in inputs.split(" ")])
print(NN.forward(x))
When you run the program it will show the dJdW value (gradient values w.r.t weights) and dJWb values (gradient values w.r.t bias weights). Then it will test the inputs on the newly trained network and print the outputs. After that, you can give the network your own inputs (between 0 and 0.5 since i trained the network to multiply inputs by 2) and it will return outputs in console. Please note that this is a highly simplified version of my real network. I want to fix the problem here before adressing it in the full version.