0
$\begingroup$

I'm trying to train a seq2seq model using PyTorch using the Multi30K dataset from Dutch to English language.

Here is my snippet of code:

CustomMulti30k class

from torchtext.datasets import Multi30k

class CustomMulti30k:
    """
    Custom class for Multi32K dataset.
    """

    def __init__(self, root, language_pair=(SRC_LANGUAGE, TGT_LANGUAGE)):

        self.train, self.valid, self.test = Multi30k(root=root, split=('train', 'valid', 'test'), language_pair=language_pair)

    def extract_sets(self):
        return self.train, self.valid, self.test

get_data function

def get_data(root='../data/.data', batch_size=BATCH_SIZE, split='train'):
    train, valid, test = CustomMulti30k(root=root).extract_sets()
    sets = {'train': train, 'valid': valid, 'test': test}
    if split in ('train', 'valid', 'test'):
        iterator = DataLoader(sets[split], batch_size=batch_size, collate_fn=CollateFn())

    else:
        raise ValueError('Split name not found !')

    return iterator

CollateFn class

from torch.nn.utils.rnn import pad_sequence
class CollateFn:

    def __init__(self):
        """
        The class constructor.
        """
        self.pad_index = PAD_IDX
        self.vocab = Vocabulary(freq_threshold=1)
        self.text_transform = self.vocab.preprocess()

    def __call__(self, batch):
        """
        Allow the class to be called as function.
        :return: source, and target as batches.
        """

        # split the batch
        src_batch, tgt_batch = [], []

        for src_sample, tgt_sample in batch:
            src_batch.append(self.text_transform[SRC_LANGUAGE](src_sample.rstrip("\n")))
            tgt_batch.append(self.text_transform[TGT_LANGUAGE](tgt_sample.rstrip("\n")))

        src_batch = pad_sequence(src_batch, padding_value=self.pad_index)
        tgt_batch = pad_sequence(tgt_batch, padding_value=self.pad_index)

        return src_batch, tgt_batch

Vocabulary class

class Vocabulary:
    """
    Builds and saves vocabulary for a language.
    """

    def __init__(self, freq_threshold=1):
        """
        The class constructor.
        :param dataset: dataset
        :param freq_threshold: int
            The frequency threshold for the processed.
        """

    self.freq_threshold = freq_threshold
    self.vocabulary = self.build_vocab()

@staticmethod
def get_tokenizer():
    """
    Get the spacy tokenizer for the lang language.
    :param lang: str
        'en' for English or 'de' for Dutch.
    :return: spacy.tokenizer
    """

    token_transform = {SRC_LANGUAGE: get_tokenizer('spacy', language=LANG_SHORTCUTS['de']),
                       TGT_LANGUAGE: get_tokenizer('spacy', language=LANG_SHORTCUTS['en'])}

    return token_transform

def _get_tokens(self, data_iterator=None, lang='de'):
    """
    Get token for an iterator containing tuple of string
    :param lang: str
        'en' or 'de' for source and target languages.
    :return: List
        List of tokens.
    """
    tokenizer = self.get_tokenizer()
    for data_sample in data_iterator:
        yield tokenizer[lang](data_sample[LANGUAGE_INDEX[lang]])

def build_vocab(self):
    """
    Build the processed of the given language.
    :return: List of Vocabs
    """
    vocabulary = {}
    for lang in [SRC_LANGUAGE, TGT_LANGUAGE]:
        data_iterator = Multi30k(root='../data/.data', split='train', language_pair=(SRC_LANGUAGE, TGT_LANGUAGE))
        vocabulary[lang] = build_vocab_from_iterator(self._get_tokens(data_iterator, lang),
                                                     min_freq=self.freq_threshold, specials=SPECIAL_SYMBOLS,
                                                     special_first=True)
        vocabulary[lang].set_default_index(UNK_IDX)

    return vocabulary

@staticmethod
def tensor_transform(tokens_idx):
    """
    Builds the representation of numericalized sentence as Tensor.

    Input : A List, [12, 1, 6, 12, 200, 100] this a transformed sentence.
    (apply itos function to get the original text-based sentence).

    Output : The same input with EOS and SOS tensor concatenated
    respectively to the end and the beginning of the input tensor.

    :param tokens_idx: List
        A transformed sentence with indices of each token in it.

    :return: Tensor
        Sentence with SOS and EOS tokens added.
    """

    return torch.cat((
        torch.tensor([SOS_IDX]),
        torch.tensor(tokens_idx),
        torch.tensor([EOS_IDX])
    ))

@staticmethod
def pipeline(*transforms):
    """
    Make a pipeline of many transformation to the given input data.

    :param transforms: List
        List of transformation as arguments to the function
    :return: Function with transformation.
    """
    def shot(sentence):
        """
        Applies transformations as input.
        :param sentence:str
        :return: Tensor
            Input as Tensor
        """
        for transform in transforms:
            sentence = transform(sentence)
        return sentence

    return shot

def postprocess(self, tensor, lang):
    """
    Postprocess a Tensor and get the corresponding text-based sentence from it.
    :return: str
        A sentence.
    """
    sentence = self.vocabulary[lang].lookup_tokens(tensor.tolist())
    return sentence

def preprocess(self):
    """
    Tokenize, numericalize and turn into tensor a sentence.

    :return: Dict
        The transformation to be applied to a text-based sentence.
    """

    sentence_transform = {}

    for lang in [SRC_LANGUAGE, TGT_LANGUAGE]:
        sentence_transform[lang] = self.pipeline(
            self.get_tokenizer()[lang],
            self.vocabulary[lang],
            self.tensor_transform
        )

    return sentence_transform

@staticmethod
def _save_file(filename, data):
    # save the processed as json
    with open(filename, 'w') as f:
        json.dump(data, f)

def save_vocabulary(self, lang=('de', 'en')):
    """
    Save processed to disk
    :return:
    """

    if 'en' not in lang and 'de' not in lang:
        raise ValueError('Not supported language(s) !')

    for language in lang:
        itos = self.vocabulary[language].get_itos()
        stoi = self.vocabulary[language].get_stoi()

        # save itos
        self._save_file(f'../data/processed/index_to_name_{language}', itos)

        # save stoi
        self._save_file(f'../data/processed/name_to_index_{language}', stoi)

def __call__(self):
    """
    Call the function when instantiation.
    :return: Set
        Set of the processed of the two languages.
    """

    self.save_vocabulary()

Training loop

if __name__ == "__main__":
    print('Training...')

    # Getting train, and valid DataLoaders
    train_iterator = get_data(root='data/.data', batch_size=BATCH_SIZE, split='test')
   
    valid_iterator = None

    # Initialize vocabulary
    vocab = Vocabulary()

    # Build vocabularies
    vocabularies = vocab.build_vocab()

    # Source and target vocabularies
    src_vocabulary = vocabularies['de']
    tgt_vocabulary = vocabularies['en']

    for epoch in range(20):
        print('len(train_iterator), type(train_iterator): ', len(train_iterator), type(train_iterator))
        for i, (src, tgt) in enumerate(train_iterator):
            print('(i, epoch) = ', i, epoch)
            print('src.shape: ', src.shape)
            print('tgt.shape: ', tgt.shape)

Output

len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
(i, epoch) =  0 0
src.shape:  torch.Size([29, 128])
tgt.shape:  torch.Size([31, 128])
(i, epoch) =  1 0
src.shape:  torch.Size([23, 128])
tgt.shape:  torch.Size([23, 128])
(i, epoch) =  2 0
src.shape:  torch.Size([27, 128])
tgt.shape:  torch.Size([31, 128])
(i, epoch) =  3 0
src.shape:  torch.Size([25, 128])
tgt.shape:  torch.Size([23, 128])
(i, epoch) =  4 0
src.shape:  torch.Size([26, 128])
tgt.shape:  torch.Size([29, 128])
(i, epoch) =  5 0
src.shape:  torch.Size([29, 128])
tgt.shape:  torch.Size([29, 128])
(i, epoch) =  6 0
src.shape:  torch.Size([29, 128])
tgt.shape:  torch.Size([35, 128])
(i, epoch) =  7 0
src.shape:  torch.Size([33, 104])
tgt.shape:  torch.Size([35, 104])
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>
len(train_iterator), type(train_iterator):  8 <class 'torch.utils.data.dataloader.DataLoader'>

So, as you can clearly see that the inner for loop get executed one time (when epoch = 0) and the that inner loop get ignored afterward (I see that like the indice to loop through the batches get freezed and not initialized to point to the first batch in the next epoch iteration).

I initialy noticed that when I used the train_loss variable initialized at the begenning of the outer scope, with train_loss = [] and when calculating average loss at the end of the outer scope, I got ZeroDivisionError: division by zero in the second iteration of outer loop because ofsum(train_loss) / len(train_loss)` and we when the second loop get not executed so no new loss value get not appended to train_loss list, hense len(train_loss) will be 0.

Full source code can be found here.

How can I fix that problem please ?

Any help or advice will be appreciated.

$\endgroup$

1 Answer 1

0
$\begingroup$

I found the issue.

if __name__ == "__main__":
    test_iter = Multi30k(split='test')

    test_dataloader = DataLoader(test_iter, batch_size=256)

    for epoch in range(1, 10):
        print('test_iter.current_pos_outer_loop: ', test_iter.current_pos)

        for (src, tgt) in test_dataloader:
            print('test_iter.current_pos: ', test_iter.current_pos)
            print('epoch: ', epoch)

The Multi30k returns a __RawTextIterableDataset instance, the latter has an attribute called current_pos which keep the number of iterated batches in the loader, this attribute get not initialized to point to the first batch in the second epoch. Hense the need to define a custom batch_sampler in the Dataloader or sampily pass an iterable Dataset to the dataloader as the dataset argument.

Here is the output from the above snippet code.

test_iter.current_pos_outer_loop:  None
test_iter.current_pos:  255
epoch:  1
test_iter.current_pos:  511
epoch:  1
test_iter.current_pos:  767
epoch:  1
test_iter.current_pos:  999
epoch:  1
test_iter.current_pos_outer_loop:  999
test_iter.current_pos_outer_loop:  999
test_iter.current_pos_outer_loop:  999
test_iter.current_pos_outer_loop:  999
test_iter.current_pos_outer_loop:  999
test_iter.current_pos_outer_loop:  999
test_iter.current_pos_outer_loop:  999
test_iter.current_pos_outer_loop:  999

As you see the current_pos index always point to the last batch after the second epoch.

from torchtext.datasets import Multi30k
from torch.utils.data import DataLoader, Dataset


if __name__ == "__main__":
    test_iter = Multi30k(split='test')

    test_dataloader = DataLoader(list(test_iter), batch_size=256, shuffle=True)

    for epoch in range(1, 10):
        for (src, tgt) in test_dataloader:
            print('epoch: ', epoch)

output

epoch:  1
epoch:  1
epoch:  1
...
epoch:  9
epoch:  9
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.