I'm trying to train a seq2seq model using PyTorch using the Multi30K dataset from Dutch to English language.
Here is my snippet of code:
CustomMulti30k class
from torchtext.datasets import Multi30k
class CustomMulti30k:
"""
Custom class for Multi32K dataset.
"""
def __init__(self, root, language_pair=(SRC_LANGUAGE, TGT_LANGUAGE)):
self.train, self.valid, self.test = Multi30k(root=root, split=('train', 'valid', 'test'), language_pair=language_pair)
def extract_sets(self):
return self.train, self.valid, self.test
get_data function
def get_data(root='../data/.data', batch_size=BATCH_SIZE, split='train'):
train, valid, test = CustomMulti30k(root=root).extract_sets()
sets = {'train': train, 'valid': valid, 'test': test}
if split in ('train', 'valid', 'test'):
iterator = DataLoader(sets[split], batch_size=batch_size, collate_fn=CollateFn())
else:
raise ValueError('Split name not found !')
return iterator
CollateFn class
from torch.nn.utils.rnn import pad_sequence
class CollateFn:
def __init__(self):
"""
The class constructor.
"""
self.pad_index = PAD_IDX
self.vocab = Vocabulary(freq_threshold=1)
self.text_transform = self.vocab.preprocess()
def __call__(self, batch):
"""
Allow the class to be called as function.
:return: source, and target as batches.
"""
# split the batch
src_batch, tgt_batch = [], []
for src_sample, tgt_sample in batch:
src_batch.append(self.text_transform[SRC_LANGUAGE](src_sample.rstrip("\n")))
tgt_batch.append(self.text_transform[TGT_LANGUAGE](tgt_sample.rstrip("\n")))
src_batch = pad_sequence(src_batch, padding_value=self.pad_index)
tgt_batch = pad_sequence(tgt_batch, padding_value=self.pad_index)
return src_batch, tgt_batch
Vocabulary class
class Vocabulary:
"""
Builds and saves vocabulary for a language.
"""
def __init__(self, freq_threshold=1):
"""
The class constructor.
:param dataset: dataset
:param freq_threshold: int
The frequency threshold for the processed.
"""
self.freq_threshold = freq_threshold
self.vocabulary = self.build_vocab()
@staticmethod
def get_tokenizer():
"""
Get the spacy tokenizer for the lang language.
:param lang: str
'en' for English or 'de' for Dutch.
:return: spacy.tokenizer
"""
token_transform = {SRC_LANGUAGE: get_tokenizer('spacy', language=LANG_SHORTCUTS['de']),
TGT_LANGUAGE: get_tokenizer('spacy', language=LANG_SHORTCUTS['en'])}
return token_transform
def _get_tokens(self, data_iterator=None, lang='de'):
"""
Get token for an iterator containing tuple of string
:param lang: str
'en' or 'de' for source and target languages.
:return: List
List of tokens.
"""
tokenizer = self.get_tokenizer()
for data_sample in data_iterator:
yield tokenizer[lang](data_sample[LANGUAGE_INDEX[lang]])
def build_vocab(self):
"""
Build the processed of the given language.
:return: List of Vocabs
"""
vocabulary = {}
for lang in [SRC_LANGUAGE, TGT_LANGUAGE]:
data_iterator = Multi30k(root='../data/.data', split='train', language_pair=(SRC_LANGUAGE, TGT_LANGUAGE))
vocabulary[lang] = build_vocab_from_iterator(self._get_tokens(data_iterator, lang),
min_freq=self.freq_threshold, specials=SPECIAL_SYMBOLS,
special_first=True)
vocabulary[lang].set_default_index(UNK_IDX)
return vocabulary
@staticmethod
def tensor_transform(tokens_idx):
"""
Builds the representation of numericalized sentence as Tensor.
Input : A List, [12, 1, 6, 12, 200, 100] this a transformed sentence.
(apply itos function to get the original text-based sentence).
Output : The same input with EOS and SOS tensor concatenated
respectively to the end and the beginning of the input tensor.
:param tokens_idx: List
A transformed sentence with indices of each token in it.
:return: Tensor
Sentence with SOS and EOS tokens added.
"""
return torch.cat((
torch.tensor([SOS_IDX]),
torch.tensor(tokens_idx),
torch.tensor([EOS_IDX])
))
@staticmethod
def pipeline(*transforms):
"""
Make a pipeline of many transformation to the given input data.
:param transforms: List
List of transformation as arguments to the function
:return: Function with transformation.
"""
def shot(sentence):
"""
Applies transformations as input.
:param sentence:str
:return: Tensor
Input as Tensor
"""
for transform in transforms:
sentence = transform(sentence)
return sentence
return shot
def postprocess(self, tensor, lang):
"""
Postprocess a Tensor and get the corresponding text-based sentence from it.
:return: str
A sentence.
"""
sentence = self.vocabulary[lang].lookup_tokens(tensor.tolist())
return sentence
def preprocess(self):
"""
Tokenize, numericalize and turn into tensor a sentence.
:return: Dict
The transformation to be applied to a text-based sentence.
"""
sentence_transform = {}
for lang in [SRC_LANGUAGE, TGT_LANGUAGE]:
sentence_transform[lang] = self.pipeline(
self.get_tokenizer()[lang],
self.vocabulary[lang],
self.tensor_transform
)
return sentence_transform
@staticmethod
def _save_file(filename, data):
# save the processed as json
with open(filename, 'w') as f:
json.dump(data, f)
def save_vocabulary(self, lang=('de', 'en')):
"""
Save processed to disk
:return:
"""
if 'en' not in lang and 'de' not in lang:
raise ValueError('Not supported language(s) !')
for language in lang:
itos = self.vocabulary[language].get_itos()
stoi = self.vocabulary[language].get_stoi()
# save itos
self._save_file(f'../data/processed/index_to_name_{language}', itos)
# save stoi
self._save_file(f'../data/processed/name_to_index_{language}', stoi)
def __call__(self):
"""
Call the function when instantiation.
:return: Set
Set of the processed of the two languages.
"""
self.save_vocabulary()
Training loop
if __name__ == "__main__":
print('Training...')
# Getting train, and valid DataLoaders
train_iterator = get_data(root='data/.data', batch_size=BATCH_SIZE, split='test')
valid_iterator = None
# Initialize vocabulary
vocab = Vocabulary()
# Build vocabularies
vocabularies = vocab.build_vocab()
# Source and target vocabularies
src_vocabulary = vocabularies['de']
tgt_vocabulary = vocabularies['en']
for epoch in range(20):
print('len(train_iterator), type(train_iterator): ', len(train_iterator), type(train_iterator))
for i, (src, tgt) in enumerate(train_iterator):
print('(i, epoch) = ', i, epoch)
print('src.shape: ', src.shape)
print('tgt.shape: ', tgt.shape)
Output
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
(i, epoch) = 0 0
src.shape: torch.Size([29, 128])
tgt.shape: torch.Size([31, 128])
(i, epoch) = 1 0
src.shape: torch.Size([23, 128])
tgt.shape: torch.Size([23, 128])
(i, epoch) = 2 0
src.shape: torch.Size([27, 128])
tgt.shape: torch.Size([31, 128])
(i, epoch) = 3 0
src.shape: torch.Size([25, 128])
tgt.shape: torch.Size([23, 128])
(i, epoch) = 4 0
src.shape: torch.Size([26, 128])
tgt.shape: torch.Size([29, 128])
(i, epoch) = 5 0
src.shape: torch.Size([29, 128])
tgt.shape: torch.Size([29, 128])
(i, epoch) = 6 0
src.shape: torch.Size([29, 128])
tgt.shape: torch.Size([35, 128])
(i, epoch) = 7 0
src.shape: torch.Size([33, 104])
tgt.shape: torch.Size([35, 104])
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator): 8 <class 'torch.utils.data.dataloader.DataLoader'>
So, as you can clearly see that the inner for loop get executed one time (when epoch = 0) and the that inner loop get ignored afterward (I see that like the indice to loop through the batches get freezed and not initialized to point to the first batch in the next epoch iteration).
I initialy noticed that when I used the train_loss variable initialized at the begenning of the outer scope, with train_loss = [] and when calculating average loss at the end of the outer scope, I got ZeroDivisionError: division by zero in the second iteration of outer loop
because ofsum(train_loss) / len(train_loss)` and we when the second loop get not executed so no new loss value get not appended to train_loss list, hense len(train_loss) will be 0.
Full source code can be found here.
How can I fix that problem please ?
Any help or advice will be appreciated.
