# Gradient Check is failing for RNN

I am trying out POS tagging using RNN but not able to figure out what's wrong in my implementation because of which the gradient check is failing. Please help. I am pasting the relevant part below:

# weights
Wxh = randn(inputLayerSize, hiddenLayerSize)*0.01; # input to hidden
Whh = randn(hiddenLayerSize, hiddenLayerSize)*0.01; # hidden to hidden
Bh = zeros(1, hiddenLayerSize); # hidden bias
Why = randn(hiddenLayerSize, outputLayerSize)*0.01; # hidden to output
By = zeros(1, outputLayerSize); # output bias

function forwardRNN(x, y, h, p, hPrev)
global Wxh, Whh, Why, Bh, By;
cost = 0;
for time in 1:length(x)
if time == 1
h[time] = tanh(x[time]'*Wxh + hPrev*Whh .+ Bh);
else
h[time] = tanh(x[time]'*Wxh + h[time-1]*Whh .+ Bh);
end
score = h[time]*Why .+ By;
p_softmax = exp(score) / sum(exp(score));
p[time] = vec(p_softmax);
cost += -sum(log(y[time]'*p[time]));
end
return cost;
end

function backwardRNN(x, y, h, p, hPrev)
global Wxh, Whh, Why, Bh, By;
dWxh = zeros(size(Wxh));
dWhh = zeros(size(Whh));
dBh = zeros(size(Bh));
dWhy = zeros(size(Why));
dBy = zeros(size(By));
dh = zeros(size(Bh));
dhnext = zeros(size(h));
for time in length(x):-1:1
dy = p[time] - y[time];
dWhy = dWhy + (dy * h[time])';
dBy = dBy + dy';
dh = (Why*dy)' + dhnext;
dhRaw = (1 - (h[time].*h[time])) .* dh;
dWxh = dWxh + (x[time] * dhRaw);
dBh = dBh + dhRaw;
if time == 1
dWhh = dWhh + (hPrev' * dhRaw);
else
dWhh = dWhh + (h[time-1]' * dhRaw);
end
dhnext = dhRaw*Whh;
end
return dWxh, dWhh, dBh, dWhy, dBy;
end

function gradCheck(inputs, targets, h, p, hPrev)
paramNameList = ["Wxh", "Whh", "Bh", "Why", "By"];
global Wxh, Whh, Why, Bh, By;
paramList = [x for x=(Wxh, Whh, Bh, Why, By)];
num_checks = 2;
delta = 1e-5;
cost = forwardRNN(inputs, targets, h, p, hPrev);
dWxh, dWhh, dBh, dWhy, dBy = backwardRNN(inputs, targets, h, p, hPrev);
dParamList = [x for x=(dWxh, dWhh, dBh, dWhy, dBy)];
for (param,dparam,name) in zip(paramList, dParamList, paramNameList)
s0 = size(dparam);
s1 = size(param);
if s0 != s1
println("Error dims dont match: ", s0," and ",s1);
end
println(name)
for i in 1:num_checks
ri = rand(1:length(param));
old_val = param[ri];
param[ri] = old_val + delta;
cg0 = forwardRNN(inputs, targets, h, p, hPrev);
param[ri] = old_val - delta;
cg1 = forwardRNN(inputs, targets, h, p, hPrev);
param[ri] = old_val
grad_numerical = (cg0 - cg1) / ( 2 * delta );
if rel_error > 1e-5
end
end
end
end


The code is in Julia programming language and is inspired from Karpathy's min-char-rnn.py

• The gradient check seems to work for the sequence of length 1 but not for 2 or more.
– lex
Mar 7, 2016 at 2:29
• I reduced a learning rate a bit and ran the training step. Got decent accuracy on train and test data. The cost vs iteration graph looks ugly though.
– lex
Mar 8, 2016 at 6:00