4
$\begingroup$

i'm trying to implement my own LSTM network. I implemented back propagation algorithm, but it doesn't pass gradient check. Can't realize where is the mistake. Please help

Here is the code problem piece of code:

def backward_propagation(self, x, y, cache):
    # T - the length of the sequence
    T = len(y)
    # perform forward propagation
    cache = self.forward_propagation(x)

    #...


    # delta for output layer
    dy = cache['y'].copy()
    dy[np.arange(len(y)), y] -= 1. # softmax loss gradient

    for t in reversed(range(T)):
        dV += np.outer(dy[t], h[t].T)
        dh[t] += self.V.T.dot(dy[t])
        dhtmp = self.V.T.dot(dy[t])
        dctmp = dct[0] * (1.0 - ct[0]**2)

        for bptt_step in np.arange(t)[::-1]:
            # add to gradients at each previous step
            do[bptt_step] = dhtmp * ct[bptt_step]
            dct[bptt_step] = dhtmp * o[bptt_step]

            dctmp += dct[bptt_step] * (1.0 - ct[bptt_step]**2)

            di[bptt_step] = dctmp * g[bptt_step]
            df[bptt_step] = dctmp * c[bptt_step-1]
            dg[bptt_step] = dctmp * i[bptt_step]

            # backprop activation functions
            diga[bptt_step] = di[bptt_step] * i[bptt_step] * (1.0 - i[bptt_step])
            dfga[bptt_step] = df[bptt_step] * f[bptt_step] * (1.0 - f[bptt_step])
            doga[bptt_step] = do[bptt_step] * o[bptt_step] * (1.0 - o[bptt_step])
            dgga[bptt_step] = dg[bptt_step] * (1.0 - g[bptt_step] ** 2)

            # backprop matrix multiply
            dWi += np.dot(diga[bptt_step].T, h[bptt_step-1])
            dWf += np.dot(dfga[bptt_step].T, h[bptt_step-1])
            dWo += np.dot(doga[bptt_step].T, h[bptt_step-1])
            dWg += np.dot(dgga[bptt_step].T, h[bptt_step-1])

            dUi[:, x[bptt_step]] += diga[bptt_step]
            dUf[:, x[bptt_step]] += dfga[bptt_step]
            dUo[:, x[bptt_step]] += doga[bptt_step]
            dUg[:, x[bptt_step]] += dgga[bptt_step]

            # update deltas for next step
            dhtmp = np.dot(self.Wi.T, diga[bptt_step])
            dhtmp += np.dot(self.Wf.T, dfga[bptt_step])
            dhtmp += np.dot(self.Wo.T, doga[bptt_step])
            dhtmp += np.dot(self.Wg.T, dgga[bptt_step])
            dctmp += dctmp * f[bptt_step]

    return [dV, dWi, dWf, dWo, dWg, dUi, dUf, dUo, dUg]

Here 'ga' suffix is for 'gate activation' - designation of gate input before non-linear activation function is applied.

$\endgroup$
  • $\begingroup$ Also i don't understand why my realization has vanishing gradient problem for the case of backprop through all previous lstm states... $\endgroup$ – ichernob Mar 7 '17 at 6:19
  • $\begingroup$ @Neil, you're right. Will share problem part of code, thanks for advice. $\endgroup$ – ichernob Mar 7 '17 at 12:08
  • $\begingroup$ Thanks for adjusting the question. There is still a fair bit of code to grep through, and sadly I cannot see any mistake in order to help you. However, I think this is now on-topic so I have retracted my close vote. $\endgroup$ – Neil Slater Mar 7 '17 at 12:43
  • $\begingroup$ @Neil, no problem. Does my gradient estimation correct? $\endgroup$ – ichernob Mar 7 '17 at 12:55
1
$\begingroup$

Hope somebody will safe lots of hours:

def backward_propagation(self, x, y, cache):
    # T - the length of the sequence
    T = len(y)
    # perform forward propagation
    cache = self.forward_propagation(x)

    #...

    # delta for output layer
    dy = cache['y'].copy()
    dy[np.arange(len(y)), y] -= 1.0 # softmax loss gradient
    # print("dy: ", dy)
    dhtmp = np.zeros((1, self.hidden_dim))
    dh_prev = np.zeros((1, self.hidden_dim))
    dctmp = np.zeros((1, self.hidden_dim))

    for t in np.arange(T)[::-1]:
        dV += np.outer(dy[t], h[t].T)
        dhtmp = self.V.T.dot(dy[t]) + dh_prev

        # add to gradients at each previous step
        do[t] = dhtmp * ct[t]
        dct[t] = dhtmp * o[t]

        dctmp += dct[t] * (1.0 - ct[t]**2)

        di[t] = dctmp * g[t]
        df[t] = dctmp * c[t-1]
        dg[t] = dctmp * i[t]

        # backprop activation functions
        diga[t] = di[t] * i[t] * (1.0 - i[t])
        dfga[t] = df[t] * f[t] * (1.0 - f[t])
        doga[t] = do[t] * o[t] * (1.0 - o[t])
        dgga[t] = dg[t] * (1.0 - g[t] ** 2)

        # backprop matrix multiply
        dWi += np.outer(diga[t], h[t-1])
        dWf += np.outer(dfga[t], h[t-1])
        dWo += np.outer(doga[t], h[t-1])
        dWg += np.outer(dgga[t], h[t-1])


        dUi[:, x[t]] += diga[t]
        dUf[:, x[t]] += dfga[t]
        dUo[:, x[t]] += doga[t]
        dUg[:, x[t]] += dgga[t]

        # update deltas for next step
        # here dh is accumulated as shared variable
        dh_prev = np.dot(self.Wi.T, diga[t])
        dh_prev += np.dot(self.Wf.T, dfga[t])
        dh_prev += np.dot(self.Wo.T, doga[t])
        dh_prev += np.dot(self.Wg.T, dgga[t])
        dctmp = dctmp * f[t]

    return [dV, dWi, dWf, dWo, dWg, dUi, dUf, dUo, dUg]
| improve this answer | |
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.