# Vanishing Gradient in a shallow network

I created an ANN in Python 3. My backpropagation algorithm seems to work up to a point where the gradient becomes very small. I am familiar with the vanishing gradient problem, but I found that it only applies to really deep network; my simple test network is no such network. It consists of an input layer (1 input node and bias), no hidden layers, and an output layer (1 output node). How do I stop the gradient from vanishing? Here is the code:

import numpy as np
from random import random

class Neural_Network(object):
def __init__(self):
# Create a simple deterministic network for testing

# Define Hyperparameters
self.inputLayerSize = 1
self.outputLayerSize = 1
self.hiddenLayerSize = 0
self.numHiddenLayer = 0
self.numExamples = 20
self.learningRate = 0.07 # LEARNING RATE
self.weightDecay = 0
# in -> out
self.weights = [] # stores matrices of each layer of weights
self.z = [] # stores matrices of each layer of weighted sums
self.a = [] # stores matrices of each layer of activity
self.biases = [] # stores all biases
self.biasNodes = []

# Biases are matrices that are added to activity matrix
# Dimensions -> numExamples_*hiddenLayerSize or numExamples_*outputLayerSize

# Biases for output layer
b = [0.5 for x in range(self.outputLayerSize)]
B = [b for x in range(self.numExamples)];
self.biases.append(np.mat(B))

# Bias nodes
b= [1 for x in range(self.numExamples)]
for i in range(self.numHiddenLayer+1):
self.biasNodes.append(np.mat(b).reshape([self.numExamples,1]))

# Weights (Parameters)
# Weight matrix between input and output layer
W = np.matrix("0.5");
self.weights.append(W)

def setBatchSize(self, numExamples):
# Changes the number of rows (examples) for biases
if (self.numExamples > numExamples):
self.biases = [b[:numExamples] for b in self.biases]

def hypTan(self, z):
# Apply hyperbolic tangent function
return (np.exp(z) - np.exp(-z)) / (np.exp(z) + np.exp(-z))

def hypTanPrime(self, z):
# Apply derivative hyperbolic tangent function
return 4/np.multiply((np.exp(z) + np.exp(-z)), (np.exp(z) + np.exp(-z)))

def forward(self, X):
# Propagate outputs through network
self.z = []
self.a = []

self.z.append(np.dot(X, self.weights) + self.biases)
self.a.append(self.hypTan(self.z))

yHat = self.a[-1]
return yHat

def backProp(self, X, y):
# Compute derivative wrt W
# out -> in
dJdWb = [] # stores matrices of each dJdWb value
dJdW = [] # stores matrices of each dJdW (equal in size to self.weights[])
delta = [] # stores matrices of each backpropagating error
result = () # stores dJdW and dJdWb
self.yHat = self.forward(X)

# Quantifying Error
print(np.linalg.norm(y-self.yHat)/np.linalg.norm(y+self.yHat))

delta.insert(0,np.multiply(-(y-self.yHat), self.hypTanPrime(self.z[-1]))) # delta = (y-yHat)(sigmoidPrime(final layer unactivated))
dJdW.insert(0, np.dot(X.T, delta) + (self.weightDecay*self.weights[-1]))
dJdWb.insert(0, np.dot(self.biasNodes[-1].T, delta) + (self.weightDecay*self.biases[-1])) # you need to backpropagate to bias nodes

result = (dJdW, dJdWb)
return result

def train(self, X, y):
for t in range(10000):
dJ = self.backProp(X, y)
dJdW = dJ
dJdWb = dJ
for i in range(len(dJdW)):
print("dJdW:", dJdW[i], sep = " ", end = "\n")
print("dJdWb:", dJdWb[i], sep = " ", end = "\n\n")
#print("Weights:", self.weights[i]);
self.weights[i] -= self.learningRate*dJdW[i]
self.biases[i] -= self.learningRate*dJdWb[i]

# Instantiating Neural Network

# Instantiating Neural Network

NN = Neural_Network() # create a deterministic NN for testing
x = np.matrix("0.025; 0.05; 0.075; 0.1; 0.125; 0.15; 0.175; 0.2; 0.225; 0.25; 0.275; 0.3; 0.325; 0.35; 0.375; 0.4; 0.425; 0.45; 0.475; 0.5")
y = np.matrix("0.05; 0.1; 0.15; 0.2; 0.25; 0.3; 0.35; 0.4; 0.45; 0.5; 0.55; 0.6; 0.65; 0.7; 0.75; 0.8; 0.85; 0.9; 0.95; 1.0")

# Training
print("INPUT: ", end = '\n')
print(x, end = '\n\n')

print("BEFORE TRAINING", NN.forward(x), sep = '\n', end = '\n\n')
print("ERROR: ")
NN.train(x,y)
print("\nAFTER TRAINING", NN.forward(x), sep = '\n', end = '\n\n')

NN.setBatchSize(1) # changing settings to receive one input at a time

while True:
inputs = input()
x = np.mat([float(i) for i in inputs.split(" ")])
print(NN.forward(x))


When you run the program it will show the dJdW value (gradient values w.r.t weights) and dJWb values (gradient values w.r.t bias weights). Then it will test the inputs on the newly trained network and print the outputs. After that, you can give the network your own inputs (between 0 and 0.5 since i trained the network to multiply inputs by 2) and it will return outputs in console. Please note that this is a highly simplified version of my real network. I want to fix the problem here before adressing it in the full version.

The way to check your gradients are correct is to test them against small weight deltas. Pick a set of weight parameters for the network. Calculate the gradients using your code. Then to test, take each weight in turn, change it by +/- $\epsilon$ and use both variants to generate cost values, use them to estimate the gradient for that weight $\frac{J_{+\epsilon} - J_{-\epsilon}}{2\epsilon}$. Use this alternative (and much slower) gradient calculation to build up a second opinion of what dJdW should be. Then compare the two values. This process is often called gradient checking.