4
$\begingroup$

I am trying out POS tagging using RNN but not able to figure out what's wrong in my implementation because of which the gradient check is failing. Please help. I am pasting the relevant part below:

# weights
Wxh = randn(inputLayerSize, hiddenLayerSize)*0.01; # input to hidden
Whh = randn(hiddenLayerSize, hiddenLayerSize)*0.01; # hidden to hidden
Bh = zeros(1, hiddenLayerSize); # hidden bias
Why = randn(hiddenLayerSize, outputLayerSize)*0.01; # hidden to output
By = zeros(1, outputLayerSize); # output bias

function forwardRNN(x, y, h, p, hPrev)
  global Wxh, Whh, Why, Bh, By;
  cost = 0;
  for time in 1:length(x)
    if time == 1
      h[time] = tanh(x[time]'*Wxh + hPrev*Whh .+ Bh);
    else
      h[time] = tanh(x[time]'*Wxh + h[time-1]*Whh .+ Bh);
    end
    score = h[time]*Why .+ By;
    p_softmax = exp(score) / sum(exp(score));
    p[time] = vec(p_softmax);
    cost += -sum(log(y[time]'*p[time]));
  end
  return cost;
end

function backwardRNN(x, y, h, p, hPrev)
  global Wxh, Whh, Why, Bh, By;
  dWxh = zeros(size(Wxh));
  dWhh = zeros(size(Whh));
  dBh = zeros(size(Bh));
  dWhy = zeros(size(Why));
  dBy = zeros(size(By));
  dh = zeros(size(Bh));
  dhnext = zeros(size(h[1]));
  for time in length(x):-1:1
    dy = p[time] - y[time];
    dWhy = dWhy + (dy * h[time])';
    dBy = dBy + dy';
    dh = (Why*dy)' + dhnext;
    dhRaw = (1 - (h[time].*h[time])) .* dh;
    dWxh = dWxh + (x[time] * dhRaw);
    dBh = dBh + dhRaw;
    if time == 1
      dWhh = dWhh + (hPrev' * dhRaw);
    else
      dWhh = dWhh + (h[time-1]' * dhRaw);
    end
    dhnext = dhRaw*Whh;
  end
  return dWxh, dWhh, dBh, dWhy, dBy;
end

# gradient checking
function gradCheck(inputs, targets, h, p, hPrev)
  paramNameList = ["Wxh", "Whh", "Bh", "Why", "By"];
  global Wxh, Whh, Why, Bh, By;
  paramList = [x for x=(Wxh, Whh, Bh, Why, By)];
  num_checks = 2;
  delta = 1e-5;
  cost = forwardRNN(inputs, targets, h, p, hPrev);
  dWxh, dWhh, dBh, dWhy, dBy = backwardRNN(inputs, targets, h, p, hPrev);
  dParamList = [x for x=(dWxh, dWhh, dBh, dWhy, dBy)];
  for (param,dparam,name) in zip(paramList, dParamList, paramNameList)
    s0 = size(dparam);
    s1 = size(param);
    if s0 != s1
      println("Error dims dont match: ", s0," and ",s1);
    end
    println(name)
    for i in 1:num_checks
      ri = rand(1:length(param));
      old_val = param[ri];
      param[ri] = old_val + delta;
      cg0 = forwardRNN(inputs, targets, h, p, hPrev);
      param[ri] = old_val - delta;
      cg1 = forwardRNN(inputs, targets, h, p, hPrev);
      param[ri] = old_val
      grad_analytic = dparam[ri];
      grad_numerical = (cg0 - cg1) / ( 2 * delta );
      rel_error = abs(grad_analytic - grad_numerical) / abs(grad_numerical + grad_analytic);
      println(grad_numerical,", ", grad_analytic, " => ",rel_error);
      if rel_error > 1e-5
        error("Gradient check failed.");
      end
      println("Gradient check passed.")
    end
  end
end

The code is in Julia programming language and is inspired from Karpathy's min-char-rnn.py

$\endgroup$
2
  • $\begingroup$ The gradient check seems to work for the sequence of length 1 but not for 2 or more. $\endgroup$
    – lex
    Mar 7 '16 at 2:29
  • $\begingroup$ I reduced a learning rate a bit and ran the training step. Got decent accuracy on train and test data. The cost vs iteration graph looks ugly though. $\endgroup$
    – lex
    Mar 8 '16 at 6:00
1
$\begingroup$

as i understand, you have the wrong backprop gradient implementation. Here you should take into account, that rnn's hidden state h has its previous state in the equation: h[time-1]. This is also must be extracted via chain rule. For more information suggest to refer this post.
It also contains Python rnn implementation.

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.