0
$\begingroup$

Python : 3.7

Tensorflow : 2.3

model :

class TranslationModel:

def __init__(self):
    # encoder layers ----
    self.sourceInputWordIdsLayer = tf.keras.layers.Input(
        shape=(maxSeqLength,), name='sourceInputWordIdsLayer', dtype=tf.int32
    )
    self.sourceInputMaskLayer = tf.keras.layers.Input(
        shape=(maxSeqLength,), name='sourceInputMaskLayer', dtype=tf.int32
    )
    self.sourceInputTypeIdsLayer = tf.keras.layers.Input(
        shape=(maxSeqLength,), name='sourceInputTypeIdsLayer', dtype=tf.int32
    )

    self.encoderInputs = dict(
        input_word_ids=self.sourceInputWordIdsLayer,
        input_mask=self.sourceInputMaskLayer,
        input_type_ids=self.sourceInputTypeIdsLayer,
    )

    self.encoderGRULayer1 = tf.keras.layers.GRU(
        stateSize, name='encoder_gru1', return_sequences=True
    )

    self.encoderGRULayer2 = tf.keras.layers.GRU(
        stateSize, name='encoder_gru2', return_sequences=True
    )

    self.encoderGRULayer3 = tf.keras.layers.GRU(
        stateSize, name='encoder_gru3', return_sequences=False
    )

    # decoder layers ----
    self.decoderInitialState = tf.keras.layers.Input(
        shape=(stateSize,), name='decoder_initial_state'
    )

    self.targetInputWordIdsLayer = tf.keras.layers.Input(
        shape=(maxSeqLength,), name='targetInputWordIdsLayer', dtype=tf.int32
    )
    self.targetInputMaskLayer = tf.keras.layers.Input(
        shape=(maxSeqLength,), name='targetInputMaskLayer', dtype=tf.int32
    )
    self.targetInputTypeIdsLayer = tf.keras.layers.Input(
        shape=(maxSeqLength,), name='targetInputTypeIdsLayer', dtype=tf.int32
    )

    self.decoderInputs = dict(
        input_word_ids=self.targetInputWordIdsLayer,
        input_mask=self.targetInputMaskLayer,
        input_type_ids=self.targetInputTypeIdsLayer,
    )

    self.decoderInitialStateInput = tf.keras.layers.Input(
        shape=(stateSize,),
        name='decoder_initial_state'
    )

    self.decoderGRULayer1 = tf.keras.layers.GRU(
        stateSize, name='decoder_gru1', return_sequences=True
    )

    self.decoderGRULayer2 = tf.keras.layers.GRU(
        stateSize, name='decoder_gru2', return_sequences=True
    )

    self.decoderGRULayer3 = tf.keras.layers.GRU(
        stateSize, name='decoder_gru3', return_sequences=True
    )

    self.decoderDenseLayer = tf.keras.layers.Dense(
        len(tokenizer.vocab), activation='softmax', name='decoder_output'
    )

def createEncoder(self):
    # connect layers to build model
    embeddings = muril_layer(self.encoderInputs)['sequence_output']

    gru1Output = self.encoderGRULayer1(embeddings)

    gru2Output = self.encoderGRULayer2(gru1Output)

    encoderOutput = self.encoderGRULayer3(gru2Output)

    return encoderOutput

def createDecoder(self, initialState):
    # connect layers to build model
    embeddings = muril_layer(self.decoderInputs)['sequence_output']

    gru1Output = self.decoderGRULayer1(embeddings, initial_state=initialState)

    gru2Output = self.decoderGRULayer2(gru1Output, initial_state=initialState)

    gru3Output = self.decoderGRULayer3(gru2Output, initial_state=initialState)

    # Connect the final dense layer that converts to
    # one-hot encoded arrays.
    decoderOutput = self.decoderDenseLayer(gru3Output)

    return decoderOutput

def makeModel(self):
    encoderOutput = self.createEncoder()

    decoderOutput = self.createDecoder(initialState=encoderOutput)

    trainModel = tf.keras.Model(
        inputs=[self.encoderInputs, self.decoderInputs],
        outputs=[decoderOutput]
    )

    encoderModel = tf.keras.Model(
        inputs=[self.encoderInputs],
        outputs=[encoderOutput]
    )

    decoderOutput = self.createDecoder(initialState=self.decoderInitialStateInput)

    decoderModel = tf.keras.Model(
        inputs=[self.decoderInputs, self.decoderInitialStateInput],
        outputs=[decoderOutput]
    )

    return trainModel, encoderModel, decoderModel

These are the training parameters:

translationModel = TranslationModel()

trainModel, encoderModel, decoderModel = translationModel.makeModel()

trainGenerator = textDataGenerator(fileList=os.path.join(pathToDataset,'train.csv'), batchSize=10)
testGenerator = textDataGenerator(fileList=os.path.join(pathToDataset,'test.csv'), batchSize=10)

optimizer = optimizers.RMSprop(lr=0.001)

trainModel.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy')

trained it using a clean dataset. loss went up to .03 and val loss went to .7

unfortunately the test results are not working as expected. giving random outputs even when using training data for prediction.

where might be the issue?

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.