1
$\begingroup$

I am implementing an attention network model over a categorical CQA dataset. Following is my code for the model:

 class QACNN(Layer):

    def __init__(self, vocab_size=MAX_NB_WORDS, embedding_size=emb_dim,filter_sizes=[1,2,3], num_filters=400, dropout_keep_prob=1.0,paras=None,learning_rate=1e-2,embeddings=None,trainable=True, **kwargs):
        self.learning_rate=learning_rate
        self.paras=paras
        self.filter_sizes=filter_sizes
        self.num_filters=num_filters
        self.dropout_keep_prob = dropout_keep_prob
        self.embeddings=embeddings


        self.embedding_size=embedding_size
        self.model_type="base"
        self.num_filters_total=self.num_filters * len(self.filter_sizes)        

        # Embedding layer
        self.updated_paras=[]
        with tf.name_scope("embedding"):
            if self.paras==None:
                if self.embeddings ==None:
                    print ("random embedding")
                    self.Embedding_W = tf.Variable(
                        tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0),
                        name="random_W")
                else:
                    self.Embedding_W = tf.Variable(np.array(self.embeddings),name="embedding_W" ,dtype="float32",trainable=trainable)
            else:
                print ("load embeddings")
                self.Embedding_W=tf.Variable(self.paras[0],trainable=trainable,name="embedding_W")
            self.updated_paras.append(self.Embedding_W)
        super(QACNN, self).__init__(** kwargs)


    def build(self, input_shape): 
            self.kernels=[]  
            self.input_dim = input_shape[-1]  
            for i, filter_size in enumerate(self.filter_sizes):
                with tf.name_scope("conv-maxpool-%s" % filter_size):
                    filter_shape = [filter_size, self.embedding_size, 1, self.num_filters]
                    if self.paras==None:
                        W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="kernel_W")
                        b = tf.Variable(tf.constant(0.1, shape=[self.num_filters]), name="kernel_b")
                        self.kernels.append((W,b))
                    else:
                        _W,_b=self.paras[1][i]
                        W=tf.Variable(_W)                
                        b=tf.Variable(_b)
                        self.kernels.append((W,b))   
                    self.updated_paras.append(W)
                    self.updated_paras.append(b)

    def compute_mask(self, inputs, mask=None):
        mask = super(QACNN, self).compute_mask(inputs, mask)
        return mask                

    def call(self,sentence, mask=None):
        embedded_chars_1 = tf.nn.embedding_lookup(self.Embedding_W, sentence)
        print "embedded::",embedded_chars_1
        embedded_chars_expanded_1 = tf.expand_dims(embedded_chars_1, -1)
        print "expanded embedded::",embedded_chars_expanded_1
        output=[]
        for i, filter_size in enumerate(self.filter_sizes): 
            conv = tf.nn.conv2d(
                embedded_chars_expanded_1,
                self.kernels[i][0],
                strides=[1, 1, 1, 1],
                padding='VALID',
                name="conv-1"
            )
            print "conv::",conv
            h = tf.nn.relu(tf.nn.bias_add(conv, self.kernels[i][1]), name="relu-1")
            print "relu::",h
            output.append(h)
        tf_reshape = tf.reshape(conv,[-1,int(h.shape[1]), int(h.shape[3])])    
        self.output_dim = [int(tf_reshape.shape[1]), int(tf_reshape.shape[2])]
        return tf_reshape

    def compute_output_shape(self, input_shape):
        newShape = list(input_shape)
        newShape[1] = self.output_dim[0]
        newShape.append(self.output_dim[1])
        return tuple(newShape)


sequence_1_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
sequence_2_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')

qns_cnn = QACNN()(sequence_1_input)
ans_cnn = QACNN()(sequence_2_input)
print qns_cnn, ans_cnn 

## calculating Similarity
qns_perm = Permute((2,1))(qns_cnn)
dens_qns = Dense(58, kernel_initializer='random_uniform')(qns_perm)
ans_perm = Permute((2,1))(ans_cnn)
qns_ans_dens = merge([dens_qns, ans_perm], mode='dot', dot_axes=1)
print qns_ans_dens

## Column max pooling
col_max = Permute((2,1))(qns_ans_dens)
col_max = GlobalMaxPooling1D()(col_max)
print col_max

## Row-max pooling
row_max = GlobalMaxPooling1D()(qns_ans_dens)
print row_max

## Attention
soft_col = Activation('softmax')(col_max)
soft_row = Activation('softmax')(row_max)

## Final Representation
qns_soft = merge([qns_cnn, soft_col], mode='dot', dot_axes=1)
ans_soft = merge([ans_cnn, soft_row], mode='dot', dot_axes=1)

## Cosine similarity
qns_ans_tensors = merge([qns_soft, ans_soft], mode='cos', dot_axes=-1)
dist = Lambda(lambda x: 1-x)(qns_ans_tensors)
dist = Dense(3,activation = 'softmax')(dist)

## Model Training
model = Model(inputs = [sequence_1_input, sequence_2_input], outputs = dist)    
print model.summary()
adagrad = Adagrad(lr = 0.1)
sgd = SGD()
adam = Adam(lr = 0.01)
model.compile(optimizer=adagrad, loss='categorical_crossentropy', metrics=['acc'])

model.fit([data_topic_body,data_cmt], score_train,
                       validation_data=[[data_topic_body_dev,data_cmt_dev], score_val],
                       epochs=80, batch_size=64)    

I was following this paper for the implementation of the above model .

It seems I am missing something here as the accuracy of the model is very poor. Am I doing something wrong here?

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.