# neural network xor gate classification

I've written a simple neural network that can predict XOR gate function. I think I've used the math correctly, but the loss doesn't go down and remains near 0.6. Can anyone help me find the reason why?

import numpy as np
import matplotlib as plt

train_X = np.array([[0,0],[0,1],[1,0],[1,1]]).T
train_Y = np.array([[0,1,1,0]])
test_X = np.array([[0,0],[0,1],[1,0],[1,1]]).T
test_Y = np.array([[0,1,1,0]])

learning_rate = 0.1
S = 5

def sigmoid(z):
return 1/(1+np.exp(-z))

def sigmoid_derivative(z):
return sigmoid(z)*(1-sigmoid(z))

S0, S1, S2 = 2, 5, 1
m = 4

w1 = np.random.randn(S1, S0) * 0.01
b1 = np.zeros((S1, 1))
w2 = np.random.randn(S2, S1) * 0.01
b2 = np.zeros((S2, 1))

for i in range(1000000):
Z1 = np.dot(w1, train_X) + b1
A1 = sigmoid(Z1)
Z2 = np.dot(w2, A1) + b2
A2 = sigmoid(Z2)

J = np.sum(-train_Y * np.log(A2) + (train_Y-1) * np.log(1-A2)) / m

dZ2 = A2 - train_Y
dW2 = np.dot(dZ2, A1.T) / m
dB2 = np.sum(dZ2, axis = 1, keepdims = True) / m
dZ1 = np.dot(w2.T, dZ2) * sigmoid_derivative(Z1)
dW1 = np.dot(dZ1, train_X.T) / m
dB1 = np.sum(dZ1, axis = 1, keepdims = True) / m

w1 = w1 - dW1 * 0.03
w2 = w2 - dW2 * 0.03
b1 = b1 - dB1 * 0.03
b2 = b2 - dB2 * 0.03

print(J)


I've solved the problem with three changes:

1. The weights are too small at the beginning, remove the scaling
2. I think the biases should also be randomly initialized:
w1 = np.random.randn(S1, S0)
b1 = np.random.randn(S1, 1)
w2 = np.random.randn(S2, S1)
b2 = np.random.randn(S2, 1)
1. Your are using a fixed learning rate of 0.03, change it to use the learning rate, and you can also increase it:

learning_rate = 0.1 ...

w1 = w1 - dW1 * learning_rate
w2 = w2 - dW2 * learning_rate
b1 = b1 - dB1 * learning_rate
b2 = b2 - dB2 * learning_rate

The image below shpws what I get for the progression of J: Just for curiosity, if we change the sigmoid function with the ReLU, take a look to what happens to J (in the code I didn't change the name of the defs...). The learning with ReLU is much faster.


def sigmoid(z):
return np.maximum(0,z)


def sigmoid_derivative(x):
x[x < = 0] = 0
x[x > 0]  = 1
return x

• Just a bit of explanation: starting with this small values for the weights, the Gradient vanishes quite early, then thre is no update after a few iterations. You will notice this effect if you plot the gradients vs. itrations after and before removing the scaling May 10 '18 at 8:08

Here's the same in tf From Github Repo Mine..

import tensorflow as tf

input_data = [[0., 0.], [0., 1.], [1., 0.], [1., 1.]]  # XOR input
output_data = [[0.], [1.], [1.], [0.]]  # XOR output

n_input = tf.placeholder(tf.float32, shape=[None, 2], name="n_input")
n_output = tf.placeholder(tf.float32, shape=[None, 1], name="n_output")

hidden_nodes = 5

b_hidden_1 = tf.Variable(tf.random_normal([hidden_nodes]), name="hidden_bias")
W_hidden_1 = tf.Variable(tf.random_normal([2, hidden_nodes]), name="hidden_weights")
hidden_1 = tf.sigmoid(tf.matmul(n_input, W_hidden) + b_hidden)

W_output = tf.Variable(tf.random_normal([hidden_nodes, 1]), name="output_weights")  # output layer's weight matrix
output = tf.sigmoid(tf.matmul(hidden, W_output))  # calc output layer's activation

cross_entropy = tf.square(n_output - output)  # simpler, but also works

loss = tf.reduce_mean(cross_entropy)  # mean the cross_entropy
optimizer = tf.train.AdamOptimizer(0.01)  # take a Adam Optimizer for optimizing with a "stepsize" of 0.01
train = optimizer.minimize(loss)  # let the optimizer train

init = tf.initialize_all_variables()

sess = tf.Session()  # create the session and therefore the graph
sess.run(init)  # initialize all variables

for epoch in range(0, 1000):
# run the training operation
cvalues = sess.run([train, loss, W_hidden, b_hidden, W_output],
feed_dict={n_input: input_data, n_output: output_data})

if epoch % 200 == 0:
print("")
print("step: {:>3}".format(epoch))
print("loss: {}".format(cvalues))

print("")
print("input: {} | output: {}".format(input_data, sess.run(output, feed_dict={n_input: [input_data]})))
print("input: {} | output: {}".format(input_data, sess.run(output, feed_dict={n_input: [input_data]})))
print("input: {} | output: {}".format(input_data, sess.run(output, feed_dict={n_input: [input_data]})))