0
$\begingroup$

I'm build a Deep Leaning model with BiLSTM and CNN for two text's semantic similarity. My data set is format as :

[s1,s2,is_similarity] with is_similarity is from 0.00 to 5.00. I want to create a set for traindata - label, but value of is_simialary is float and I dont know how to convert it to label for every sentences pair. My code is: Dataset.py

def create_train_dev_set(tokenizer, sentences_pair, is_similar, max_sequence_length, validation_split_ratio):
"""
Create training and validation dataset
Args:
    tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object
    sentences_pair (list): list of tuple of sentences pairs
    is_similar (list): list containing labels if respective sentences in sentence1 and sentence2
                       are same or not (1 if same else 0)
    max_sequence_length (int): max sequence length of sentences to apply padding
    validation_split_ratio (float): contain ratio to split training data into validation data

Returns:
    train_data_1 (list): list of input features for training set from sentences1
    train_data_2 (list): list of input features for training set from sentences2
    labels_train (np.array): array containing similarity score for training data
    leaks_train(np.array): array of training leaks features

    val_data_1 (list): list of input features for validation set from sentences1
    val_data_2 (list): list of input features for validation set from sentences1
    labels_val (np.array): array containing similarity score for validation data
    leaks_val (np.array): array of validation leaks features
"""
sentences1 = [x[0].lower() for x in sentences_pair]
sentences2 = [x[1].lower() for x in sentences_pair]
train_sequences_1 = tokenizer.texts_to_sequences(sentences1)
train_sequences_2 = tokenizer.texts_to_sequences(sentences2)
leaks = [[len(set(x1)), len(set(x2)), len(set(x1).intersection(x2))]
         for x1, x2 in zip(train_sequences_1, train_sequences_2)]

train_padded_data_1 = pad_sequences(train_sequences_1, maxlen=max_sequence_length)
train_padded_data_2 = pad_sequences(train_sequences_2, maxlen=max_sequence_length)
#label
#train_labels = np.array(is_similar)
train_labels = np.zeros((len(is_similar), 6))
for i, label in enumerate(is_similar):
        if np.floor(label) + 1 < 6:
            train_labels[i, int(np.floor(label)) + 1] = label - np.floor(label)
        train_labels[i, int(np.floor(label))] = np.floor(label) - label + 1

leaks = np.array(leaks)

shuffle_indices = np.random.permutation(np.arange(len(train_labels)))
train_data_1_shuffled = train_padded_data_1[shuffle_indices]
train_data_2_shuffled = train_padded_data_2[shuffle_indices]
train_labels_shuffled = train_labels[shuffle_indices]
leaks_shuffled = leaks[shuffle_indices]

dev_idx = max(1, int(len(train_labels_shuffled) * validation_split_ratio))

del train_padded_data_1
del train_padded_data_2
gc.collect()

train_data_1, val_data_1 = train_data_1_shuffled[:-dev_idx], train_data_1_shuffled[-dev_idx:]
train_data_2, val_data_2 = train_data_2_shuffled[:-dev_idx], train_data_2_shuffled[-dev_idx:]
labels_train, labels_val = train_labels_shuffled[:-dev_idx], train_labels_shuffled[-dev_idx:]
leaks_train, leaks_val = leaks_shuffled[:-dev_idx], leaks_shuffled[-dev_idx:]

return train_data_1, train_data_2, labels_train, leaks_train, val_data_1, val_data_2, labels_val, leaks_val

and my model is:

class SiameseBiLSTM:
def __init__(self, embedding_dim, max_sequence_length, number_lstm, number_dense, rate_drop_lstm,
             rate_drop_dense, hidden_activation, validation_split_ratio,c):
    self.c = c
    self.embedding_dim = embedding_dim
    self.max_sequence_length = max_sequence_length
    self.number_lstm_units = number_lstm
    self.rate_drop_lstm = rate_drop_lstm
    self.number_dense_units = number_dense
    self.activation_function = hidden_activation
    self.rate_drop_dense = rate_drop_dense
    self.validation_split_ratio = validation_split_ratio
def _lossfunction(self,y_true,y_pred):
    ny_true = y_true[:,1] + 2*y_true[:,2] + 3*y_true[:,3] + 4*y_true[:,4] + 5*y_true[:,5]
    ny_pred = y_pred[:,1] + 2*y_pred[:,2] + 3*y_pred[:,3] + 4*y_pred[:,4] + 5*y_pred[:,5]
    my_true = K.mean(ny_true)
    my_pred = K.mean(ny_pred)
    var_true = (ny_true - my_true)**2
    var_pred = (ny_pred - my_pred)**2
    return -K.sum((ny_true-my_true)*(ny_pred-my_pred),axis=-1) / (K.sqrt(K.sum(var_true,axis=-1)*K.sum(var_pred,axis=-1)))


def train_model(self, sentences_pair, is_similar, embedding_meta_data, model_save_directory='./'):
    tokenizer, embedding_matrix = embedding_meta_data[
        'tokenizer'], embedding_meta_data['embedding_matrix']

    train_data_x1, train_data_x2, train_labels, leaks_train, \
    val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer, sentences_pair,
                                                                           is_similar, self.max_sequence_length,
                                                                           self.validation_split_ratio)

    if train_data_x1 is None:
        print("++++ !! Failure: Unable to train model ++++")
        return None

    nb_words = len(tokenizer.word_index) + 1

     # Creating word embedding layer
    embedding_layer = Embedding(nb_words, self.embedding_dim, weights=[embedding_matrix],
                                input_length=self.max_sequence_length, trainable=False)

    # Creating LSTM Encoder
    lstm_layer = Bidirectional(LSTM(self.number_lstm_units, dropout=self.rate_drop_lstm,
                               recurrent_dropout=self.rate_drop_lstm, return_sequences=True,activation='relu'))

    # Creating LSTM Encoder layer for First Sentence

    sequence_1_input = Input(
        shape=(self.max_sequence_length,), dtype='int32')
    embedded_sequences_1 = embedding_layer(sequence_1_input)
    x1 = lstm_layer(embedded_sequences_1)

    # Creating LSTM Encoder layer for Second Sentence
    sequence_2_input = Input(
        shape=(self.max_sequence_length,), dtype='int32')
    embedded_sequences_2 = embedding_layer(sequence_2_input)
    x2 = lstm_layer(embedded_sequences_2)

    # Creating leaks input
    leaks_input = Input(shape=(leaks_train.shape[1],))
    leaks_dense = Dense(int(self.number_dense_units/2), activation=self.activation_function)(leaks_input)

    Convolt_Layer=[]
    MaxPool_Layer=[]
    Flatten_Layer=[]
    for kernel_size, filters in self.c['cnnfilters'].items():
        Convolt_Layer.append(Convolution1D(filters=filters,
                                           kernel_size=kernel_size,
                                           padding='valid',
                                           activation=self.c['cnnactivate'],
                                           kernel_initializer=self.c['cnninitial']))
        MaxPool_Layer.append(MaxPooling1D(pool_size=int(self.c['sentencepad']-kernel_size+1)))
        Flatten_Layer.append(Flatten())
        print("Kernel size: ",kernel_size)
    Convolted_tensor0=[]
    Convolted_tensor1=[]
    for channel in range(len(self.c['cnnfilters'])):
        Convolted_tensor0.append(Convolt_Layer[channel](x1))
        Convolted_tensor1.append(Convolt_Layer[channel](x2))
    MaxPooled_tensor0=[]
    MaxPooled_tensor1=[]
    for channel in range(len(self.c['cnnfilters'])):
        MaxPooled_tensor0.append(MaxPool_Layer[channel](Convolted_tensor0[channel]))
        MaxPooled_tensor1.append(MaxPool_Layer[channel](Convolted_tensor1[channel]))
    Flattened_tensor0=[]
    Flattened_tensor1=[]
    for channel in range(len(self.c['cnnfilters'])):
        Flattened_tensor0.append(Flatten_Layer[channel](MaxPooled_tensor0[channel]))
        Flattened_tensor1.append(Flatten_Layer[channel](MaxPooled_tensor1[channel]))
    if len(self.c['cnnfilters']) > 1:
        Flattened_tensor0=concatenate(Flattened_tensor0)
        Flattened_tensor1=concatenate(Flattened_tensor1)
    else:
        Flattened_tensor0=Flattened_tensor0[0]
        Flattened_tensor1=Flattened_tensor1[0]
    absDifference = Lambda(lambda X:K.abs(X[0] - X[1]))([Flattened_tensor0,Flattened_tensor1])
    mulDifference = multiply([Flattened_tensor0,Flattened_tensor1])
    allDifference = concatenate([absDifference,mulDifference])
    for ilayer, densedimension in enumerate(self.c['densedimension']):
        allDifference = Dense(units=int(densedimension), 
                              activation=self.c['denseactivate'], 
                              kernel_initializer=self.c['denseinitial'])(allDifference)
    output = Dense(name='output',
                   units=self.c['num_classes'],
                   activation='softmax', 
                   kernel_initializer=self.c['denseinitial'])(allDifference)
    model = Model(inputs=[sequence_1_input, sequence_2_input, leaks_input], outputs=output)
    model.compile(loss={'output': self._lossfunction}, optimizer=self.c['optimizer'])

    early_stopping = EarlyStopping(monitor='val_loss', patience=3)

    STAMP = 'lstm_%d_%d_%.2f_%.2f' % (
        self.number_lstm_units, self.number_dense_units, self.rate_drop_lstm, self.rate_drop_dense)

    checkpoint_dir = model_save_directory + \
        'checkpoints/' + str(int(time.time())) + '/'

    if not os.path.exists(checkpoint_dir):
        os.makedirs(checkpoint_dir)

    bst_model_path = checkpoint_dir + STAMP + '.h5'

    model_checkpoint = ModelCheckpoint(
        bst_model_path, save_best_only=True, save_weights_only=False)

    tensorboard = TensorBoard(
        log_dir=checkpoint_dir + "logs/{}".format(time.time()))

    model.fit([train_data_x1, train_data_x2, leaks_train], train_labels,
              validation_data=(
                  [val_data_x1, val_data_x2, leaks_val], val_labels),
              epochs=20, batch_size=200, shuffle=True,
              callbacks=[early_stopping, model_checkpoint, tensorboard])
    # plot_model(model, to_file='model_plot.png',
    #            show_shapes=True, show_layer_names=True)
    return bst_model_path

my code update is: Text similarity

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.