5
$\begingroup$

From the comments in my previous question, I'm trying to build my own custom weight initializer for an RNN. Based on the code given here (careful - the updated version of Keras uses 'initializers' instead of 'initializations' according to fchollet), I've put together an attempt.

import numpy as np
import pandas, math, sys, keras
from keras import optimizers
from keras import initializers
from keras.models import Sequential
from keras.layers import Dense, SimpleRNN
from keras.regularizers import l2
import numpy as np

def rnn_model(hid_dim=10, ker_reg=0.01, rec_reg=0.01, optimizer="sgd", w_mean = 0., w_var = 0.05):
        my_init = lambda shape: initializers.TruncatedNormal(mean=w_mean, stddev=w_var)
        model = Sequential()
        model.add(SimpleRNN(units=hid_dim, activation='relu', kernel_initializer=my_init, recurrent_initializer=my_init, input_shape = (X.shape[1], X.shape[2]), kernel_regularizer=l2(ker_reg), recurrent_regularizer = l2(rec_reg), return_sequences = False))
        model.add(Dense(Y.shape[1], activation='softmax'))
        model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
        print 'fitting a model'
        return model

When I call rnn_model later, I get an error:

    model = rnn_model(hid_dim=hid_val, ker_reg=ker_reg_best, rec_reg=rec_reg_best, optimizer=optim, w_mean=ave_weights, w_var=var_weights)
  File "rnn.py", line 187, in rnn_model
    model.add(SimpleRNN(units=hid_dim, activation='relu', kernel_initializer=my_init, recurrent_initializer=my_init, input_shape = (X.shape[1], X.shape[2]), kernel_regularizer=l2(ker_reg), recurrent_regularizer = l2(rec_reg), return_sequences = False))
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/models.py", line 430, in add
    layer(x)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/layers/recurrent.py", line 257, in __call__
    return super(Recurrent, self).__call__(inputs, **kwargs)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 551, in __call__
    self.build(input_shapes[0])
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/layers/recurrent.py", line 478, in build
    constraint=self.kernel_constraint)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 384, in add_weight
    weight = K.variable(initializer(shape), dtype=K.floatx(), name=name)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 288, in variable
    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 197, in __init__
    expected_shape=expected_shape)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 274, in _init_from_args
    initial_value(), name="initial_value", dtype=dtype)
TypeError: __call__() takes at least 2 arguments (1 given)

Does anyone know how to initialize a Keras model using custom parameters for the initializer?

$\endgroup$
0
1
$\begingroup$

Check this example (a reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" by Quoc V. Le, Navdeep Jaitly, Geoffrey E. Hinton).

Relevant code:

model.add(SimpleRNN(hidden_units,
                    kernel_initializer=initializers.RandomNormal(stddev=0.001),
                    recurrent_initializer=initializers.Identity(gain=1.0),
                    activation='relu',
                    input_shape=x_train.shape[1:]))
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.