When training with 50 thousand pairs of questions and loss 0.2 accuracy 0.9 it does not give adequate answers

#@title import
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import Input, LSTM, Dense, Embedding, Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import TensorBoard
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Masking
from tensorflow.keras.layers import GlobalAveragePooling1D
from tensorflow.keras.layers import Attention
from tensorflow.keras.callbacks import TensorBoard
import matplotlib.pyplot as plt

#@title класс нейросети

class PrintLogsCallback(tf.keras.callbacks.Callback):
    def on_train_batch_end(self, batch, logs=None):
        if batch % 100 == 0:
            print(f'Batch {batch} - loss: {logs["loss"]:.4f}')

class Seq2SeqChatbot:
    def __init__(self,tokenizer_inputs, tokenizer_outputs, max_input_length, max_output_length, num_encoder_tokens, num_decoder_tokens, latent_dim):
        self.tokenizer_inputs = tokenizer_inputs
        self.tokenizer_outputs = tokenizer_outputs
        self.max_input_length = max_input_length
        self.max_output_length = max_output_length
        self.num_encoder_tokens = num_encoder_tokens
        self.num_decoder_tokens = num_decoder_tokens
        self.latent_dim = latent_dim

        # Encoder
        encoder_inputs = Input(shape=(None,))
        encoder_embedding = Embedding(self.num_encoder_tokens, self.latent_dim)(encoder_inputs)
        encoder_outputs, state_h, state_c = LSTM(self.latent_dim, return_state=True)(encoder_embedding)
        self.encoder_model = Model(inputs=encoder_inputs, outputs=[encoder_outputs, state_h, state_c])

        # Decoder
        decoder_inputs = Input(shape=(None,))
        decoder_embedding = Embedding(self.num_encoder_tokens, self.latent_dim)(decoder_inputs)
        decoder_lstm = LSTM(self.latent_dim, return_sequences=True, return_state=True)
        decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=[state_h, state_c])
        decoder_attention = Attention()([decoder_outputs, encoder_outputs])
        decoder_dense = Dense(num_decoder_tokens, activation='softmax')(decoder_attention)
        decoder_dense = Dense(num_decoder_tokens, activation='softmax')
        decoder_outputs = decoder_dense(decoder_outputs)
        self.decoder_model = Model(inputs=[decoder_inputs, state_h, state_c], outputs=[decoder_outputs])

        # Full model
        encoder_input = Input(shape=(None,))
        decoder_input = Input(shape=(None,))
        encoder_output, encoder_h, encoder_c = self.encoder_model(encoder_input)
        decoder_output = self.decoder_model([decoder_input, encoder_h, encoder_c])
        self.model = Model(inputs=[encoder_input, decoder_input], outputs=[decoder_output])

    def train(self, input_data, output_data, batch_size, epochs):
        # Compile the model
        self.model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

        # Create callback to print training logs
        print_logs_callback = PrintLogsCallback()

        # Train the model
        self.model.fit(x=[input_data, output_data[:, :-1]], y=output_data[:, 1:], batch_size=batch_size, epochs=epochs, callbacks=[print_logs_callback])
        encoder_output, encoder_h, encoder_c = self.encoder_model.predict(np.array([self.tokenizer_inputs.texts_to_sequences(["привет,"])[0]]))

    def save(self, filepath):

    def load(self, filepath):
        self.model = tf.keras.models.load_model(filepath)

    def reply(self, input_text):
        # Tokenize the input text
        input_seq = self.tokenizer_inputs.texts_to_sequences([input_text])[0]

        # Encode the input sequence
        encoder_output, encoder_h, encoder_c = self.encoder_model.predict(np.array([input_seq]), verbose = 0)

        # Initialize the target sequence with a start token
        target_seq = np.zeros((1, 1))
        target_seq[0, 0] = self.tokenizer_outputs.word_index['<start>']

        # Initialize the decoded sequence with an empty string
        decoded_seq = ''

        # Initialize a flag to stop decoding when an end token is generated
        end_of_sequence = False

        # Set the maximum output length
        max_output_length = self.max_output_length

        # Start decoding the target sequence
        while not end_of_sequence:
            # Generate the next token in the target sequence
            decoder_outputs = self.decoder_model.predict([target_seq, encoder_h, encoder_c], verbose = 0)
            token_index = np.argmax(decoder_outputs[0, 0, :])
            print(target_seq[0, 0])

            # Map the token index to the corresponding word
            decoded_word = self.tokenizer_outputs.index_word[token_index]

            # Append the decoded word to the decoded sequence

            # Stop decoding when an end token is generated
            if decoded_word == '<end>' or len(decoded_seq) >= max_output_length:
                end_of_sequence = True
                # Update the target sequence with the next token
                target_seq[0, 0] = token_index
                decoded_seq += decoded_word + ' '

        # Return the decoded sequence
        return decoded_seq.strip()

def load_dataset(filename, num_samples):
  input_texts = []
  target_texts = []
  tokenizer = Tokenizer(filters='')
  with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
    lines = f.read().split('\n')
  for i in range(0, len(lines)-1, 2):
      input_text = lines[i]
      target_text = lines[i+1]
      if (i+1 >= num_samples):
  input_texts = ['<start> ' + text + ' <end>' for text in input_texts]
  target_texts = ['<start> ' + text + ' <end>' for text in target_texts]
  tokenizer.fit_on_texts(input_texts + target_texts)
  input_sequences = tokenizer.texts_to_sequences(input_texts)
  target_sequences = tokenizer.texts_to_sequences(target_texts)

  max_input_length = max(len(seq) for seq in input_sequences)
  max_target_length = max(len(seq) for seq in target_sequences)

  encoder_input_data = pad_sequences(input_sequences, maxlen=max_input_length, padding='post')
  decoder_input_data = pad_sequences(target_sequences, maxlen=max_target_length, padding='post')
  decoder_target_data = np.zeros((num_samples, max_target_length, len(tokenizer.word_index)+1), dtype=np.float32)
  for i, target_sequence in enumerate(target_sequences):
      for t, word in enumerate(target_sequence):
          if t > 0:
              decoder_target_data[i, t-1, word] = 1.0

  return (encoder_input_data, decoder_input_data, decoder_target_data, tokenizer, tokenizer, max_input_length, max_target_length)

num_samples = 20000
input_data, output_data, _, tokenizer_inputs, tokenizer_outputs, max_input_length, max_output_length = load_dataset('dataset.txt', num_samples)


chatbot = Seq2SeqChatbot(tokenizer_inputs, tokenizer_outputs, max_input_length, max_output_length, input_vocab_size, output_vocab_size, 256)

chatbot.train(input_data=input_data, output_data=output_data, batch_size=400, epochs=30)

while True:
    input_text = input('You: ')
    if input_text.lower() in ['exit', 'quit']:
    response = chatbot.reply(input_text)
    print('Bot:', response)

Judging by the decoder_output the model is trained, but for some reason the answers are the same

  • $\begingroup$ Your training data seems too little; either use it to fine-tune a pre-trained model or use much more data. Also, you should use beam search decoding instead of greedy decoding. Apart from that, you should use validation data to understand if you model is overfitting. $\endgroup$
    – noe
    Commented Feb 22, 2023 at 7:33
  • $\begingroup$ @noe I trained it on 300,000 pairs, got 0.3 loss and accuracy of 0.95, in the validation the same result, but the words are not related to the input text and to each other, I do not understand why. maybe latent_dim 64 is not enough for it to understand? $\endgroup$
    – vamperg
    Commented Feb 22, 2023 at 17:15


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.