0
$\begingroup$

I have trained the network with data in batches of some batch size >1. After training, I am using trained network and then manually update parameters for every example using backpropagation. The following is the code I have written in Keras for updates of layer weights. Although there is no error, but after backpropagation on 10 examples, I see that the prediction is a zigzag. I do not know what is the conceptual error.

def layer_update(lr,numlayer,C,Wh,loop_counter,hprevRecv,cprevRecv,hcurrentRecv,ccurrentRecv):
Wlayer=model.layers[numlayer].get_weights()[0] # extract model weights
Ulayer=model.layers[numlayer].get_weights()[1]
blayer=model.layers[numlayer].get_weights()[2]
neuL=int(len(blayer)/4)
W_ix=Wlayer[:,:neuL]
W_fx=Wlayer[:,neuL:2*neuL]
W_cx=Wlayer[:,2*neuL:3*neuL]
W_o=Wlayer[:,3*neuL:]
U_ix=Ulayer[:,:neuL]
U_fx=Ulayer[:,neuL:2*neuL]
U_cx=Ulayer[:,2*neuL:3*neuL]
U_o=Ulayer[:,3*neuL:]
b_ix=blayer[:neuL]
b_fx=blayer[neuL:2*neuL]
b_cx=blayer[2*neuL:3*neuL]
b_o=blayer[3*neuL:]
get_layer_output = K.function([model.layers[0].input,model.layers[1].input],[model.layers[numlayer-1].output])
ilayer = get_layer_output(trainX[len(trainX)-batch_size:len(trainX),:])[0]
ilayer=ilayer[-1,-1,:]
ilayer=numpy.expand_dims(ilayer,axis=0)  # input from previouslayer
intermediate_layer_model = Model(model.layers[0].input,outputs=model.layers[numlayer].output)
intermediate_output = intermediate_layer_model.predict(trainX[len(trainX)-batch_size:len(trainX),:],batch_size=batch_size)

if(loop_counter==0):
    hprev=intermediate_output[1][batch_size-2:batch_size-1]  # h_{t-1}
    cprev=intermediate_output[2][batch_size-2:batch_size-1]
    hcurrent=intermediate_output[1][batch_size-1]
    ccurrent=intermediate_output[2][batch_size-1]
else:
    hprev=hprevRecv
    cprev=cprevRecv
    hcurrent=hcurrentRecv
    ccurrent=ccurrentRecv

if(hprev.shape[0]!=1):
    hprev=numpy.expand_dims(hprev,axis=0)
    cprev=numpy.expand_dims(cprev,axis=0)
hprev=numpy.transpose(hprev)
i_gate=sigmoidf(numpy.matmul(ilayer,W_ix)+numpy.transpose(numpy.matmul(U_ix,hprev))+b_ix)
f_gate=sigmoidf(numpy.matmul(ilayer,W_fx)+numpy.transpose(numpy.matmul(U_fx,hprev))+b_fx)
o_gate=sigmoidf(numpy.matmul(ilayer,W_o)+numpy.transpose(numpy.matmul(U_o,hprev))+b_o)
c_tilde=numpy.tanh(numpy.matmul(ilayer,W_cx)+numpy.transpose(numpy.matmul(U_cx,hprev))+b_cx)    


intermediate_layer_model = Model(model.layers[0].input,outputs=model.layers[numlayer+1].output)   
earlier_output = intermediate_layer_model.predict(trainX[len(trainX)-batch_size:len(trainX),:],batch_size=batch_size)
earlier_output=numpy.expand_dims(earlier_output[-1],axis=1)
new_output=numpy.matmul(Wh,numpy.transpose(dataset[train_size:train_size+1,:]))
change=new_output-earlier_output
change1=change[:neuL,:]

print('hc',hcurrent.shape)    
deltawix=numpy.matmul(numpy.transpose(ilayer),numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2),c_tilde),i_gate),(1-i_gate))))
deltawfx=numpy.matmul(numpy.transpose(ilayer),numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2),cprev),f_gate),(1-f_gate))))
deltawo=numpy.matmul(numpy.transpose(ilayer),numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.tanh(ccurrent),numpy.multiply(o_gate,(1-o_gate)))))
deltawcx=numpy.matmul(numpy.transpose(ilayer),numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2,i_gate),1-numpy.tanh(numpy.matmul(ilayer,W_cx)+numpy.transpose(numpy.matmul(U_cx,hprev))+b_cx)**2)))

deltauix=numpy.matmul(hprev,numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2),c_tilde),i_gate),(1-i_gate))))
deltaufx=numpy.matmul(hprev,numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2),cprev),f_gate),(1-f_gate))))
deltauo=numpy.matmul(hprev,numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.tanh(ccurrent),numpy.multiply(o_gate,(1-o_gate)))))
deltaucx=numpy.matmul(hprev,numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2,i_gate),1-numpy.tanh(numpy.matmul(ilayer,W_cx)+numpy.transpose(numpy.matmul(U_cx,hprev))+b_cx)**2)))

deltabix=numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2),c_tilde),i_gate),(1-i_gate)))
deltabfx=numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2),cprev),f_gate),(1-f_gate)))
deltabo=numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.tanh(ccurrent),numpy.multiply(o_gate,(1-o_gate))))
deltabcx=numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2,i_gate),1-numpy.tanh(numpy.matmul(ilayer,W_cx)+numpy.transpose(numpy.matmul(U_cx,hprev))+b_cx)**2))


temp=numpy.multiply(numpy.transpose(change1),numpy.multiply(numpy.multiply(numpy.multiply(numpy.multiply(o_gate,1-numpy.tanh(ccurrent)**2),c_tilde),i_gate),(1-i_gate)))

W_ix=W_ix-lr*deltawix
W_fx=W_fx-lr*deltawfx
W_o=W_o-lr*deltawo
W_cx=W_cx-lr*deltawcx
U_ix=U_ix-lr*deltauix
U_fx=U_fx-lr*deltaufx
U_o=U_o-lr*deltauo
U_cx=U_cx-lr*deltaucx
b_ix=b_ix-lr*deltabix
b_fx=b_fx-lr*deltabfx
b_o=b_o-lr*deltabo
b_cx=b_cx-lr*deltabcx
Wii=numpy.concatenate([W_ix,W_fx,W_o,W_cx],axis=1)
Uii=numpy.concatenate([U_ix,U_fx,U_o,U_cx],axis=1)
bii=numpy.concatenate([b_ix,b_fx,b_o,b_cx],axis=1)
hprev=numpy.copy(hcurrent)
cprev=numpy.copy(ccurrent)
hprev=numpy.transpose(hprev)
i_gate=sigmoidf(numpy.matmul(ilayer,W_ix)+numpy.transpose(numpy.matmul(U_ix,hprev))+b_ix)
f_gate=sigmoidf(numpy.matmul(ilayer,W_fx)+numpy.transpose(numpy.matmul(U_fx,hprev))+b_fx)
o_gate=sigmoidf(numpy.matmul(ilayer,W_o)+numpy.transpose(numpy.matmul(U_o,hprev))+b_o)
c_tilde=numpy.tanh(numpy.matmul(ilayer,W_cx)+numpy.transpose(numpy.matmul(U_cx,hprev))+b_cx) 
ccurrent=numpy.multiply(c_tilde,i_gate)+numpy.multiply(f_gate,cprev)
hcurrent=numpy.multiply(o_gate,numpy.tanh(ccurrent))
if(hcurrent.shape[0]==1):
    hcurrentpass=hcurrent[0]
    ccurrentpass=ccurrent[0]
else:
    hcurrentpass=numpy.copy(hcurrent)
    ccurrentpass=numpy.copy(ccurrent)
print('hcshape',hcurrent.shape)

return W_ix,temp,Wii,Uii,bii,hprev,cprev,hcurrentpass,ccurrentpass
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.