Custom weight initialization in Keras

Question

From the comments in my previous question, I'm trying to build my own custom weight initializer for an RNN. Based on the code given here (careful - the updated version of Keras uses 'initializers' instead of 'initializations' according to fchollet), I've put together an attempt.

import numpy as np
import pandas, math, sys, keras
from keras import optimizers
from keras import initializers
from keras.models import Sequential
from keras.layers import Dense, SimpleRNN
from keras.regularizers import l2
import numpy as np

def rnn_model(hid_dim=10, ker_reg=0.01, rec_reg=0.01, optimizer="sgd", w_mean = 0., w_var = 0.05):
        my_init = lambda shape: initializers.TruncatedNormal(mean=w_mean, stddev=w_var)
        model = Sequential()
        model.add(SimpleRNN(units=hid_dim, activation='relu', kernel_initializer=my_init, recurrent_initializer=my_init, input_shape = (X.shape[1], X.shape[2]), kernel_regularizer=l2(ker_reg), recurrent_regularizer = l2(rec_reg), return_sequences = False))
        model.add(Dense(Y.shape[1], activation='softmax'))
        model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
        print 'fitting a model'
        return model

When I call rnn_model later, I get an error:

    model = rnn_model(hid_dim=hid_val, ker_reg=ker_reg_best, rec_reg=rec_reg_best, optimizer=optim, w_mean=ave_weights, w_var=var_weights)
  File "rnn.py", line 187, in rnn_model
    model.add(SimpleRNN(units=hid_dim, activation='relu', kernel_initializer=my_init, recurrent_initializer=my_init, input_shape = (X.shape[1], X.shape[2]), kernel_regularizer=l2(ker_reg), recurrent_regularizer = l2(rec_reg), return_sequences = False))
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/models.py", line 430, in add
    layer(x)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/layers/recurrent.py", line 257, in __call__
    return super(Recurrent, self).__call__(inputs, **kwargs)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 551, in __call__
    self.build(input_shapes[0])
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/layers/recurrent.py", line 478, in build
    constraint=self.kernel_constraint)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 384, in add_weight
    weight = K.variable(initializer(shape), dtype=K.floatx(), name=name)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py", line 288, in variable
    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 197, in __init__
    expected_shape=expected_shape)
  File "/user/pkgs/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 274, in _init_from_args
    initial_value(), name="initial_value", dtype=dtype)
TypeError: __call__() takes at least 2 arguments (1 given)

Does anyone know how to initialize a Keras model using custom parameters for the initializer?

Answer 1

Check this example (a reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" by Quoc V. Le, Navdeep Jaitly, Geoffrey E. Hinton).

Relevant code:

model.add(SimpleRNN(hidden_units,
                    kernel_initializer=initializers.RandomNormal(stddev=0.001),
                    recurrent_initializer=initializers.Identity(gain=1.0),
                    activation='relu',
                    input_shape=x_train.shape[1:]))

Answer 2

This is the example given in the keras source code from initializers_v2.py

    import tensorflow as tf

    class ExampleRandomNormal(tf.keras.initializers.Initializer):

      def __init__(self, mean, stddev):
        self.mean = mean
        self.stddev = stddev

      def __call__(self, shape, dtype=None, **kwargs):
        return tf.random.normal(
            shape, mean=self.mean, stddev=self.stddev, dtype=dtype)

      def get_config(self):  # To support serialization
        return {"mean": self.mean, "stddev": self.stddev}

it can be easily adapted. An example of which ...

class myWeightInitializer(tf.keras.initializers.Initializer):

    def __call__(self, shape, dtype=None, **kwargs):
        tf.print(shape)

        outputs = shape[0]
        inputs = shape[1]

        vals = np.zeros(shape)

        for y in range(outputs):
            yy = y / (outputs-1.)
            for x in range(inputs):
                xx = x / (inputs-1.)

                d = abs(xx-yy) * 2
                if d > 1:
                    d = 1

                d = 1 - d;

                vals[y, x] = d * d / outputs

        return tf.convert_to_tensor(vals, dtype)

      

then use kernel_initializer=myWeightInitializer() as a parameter to a layer