Enable Mini-batch Processing on PyTorch Word Embeddings

Question

I am new to PyTorch and trying to create word embeddings. I started with the example below and everything works fine and it completes relatively quickly.

CONTEXT_SIZE = 2
EMBEDDING_DIM = 10
# We will use Shakespeare Sonnet 2
test_sentence = """When forty winters shall besiege thy brow,
And dig deep trenches in thy beauty's field,
Thy youth's proud livery so gazed on now,
Will be a totter'd weed of small worth held:
Then being asked, where all thy beauty lies,
Where all the treasure of thy lusty days;
To say, within thine own deep sunken eyes,
Were an all-eating shame, and thriftless praise.
How much more praise deserv'd thy beauty's use,
If thou couldst answer 'This fair child of mine
Shall sum my count, and make my old excuse,'
Proving his beauty by succession thine!
This were to be new made when thou art old,
And see thy blood warm when thou feel'st it cold.""".split()
# we should tokenize the input, but we will ignore that for now
# build a list of tuples.  Each tuple is ([ word_i-2, word_i-1 ], target word)
trigrams = [([test_sentence[i], test_sentence[i + 1]], test_sentence[i + 2])
            for i in range(len(test_sentence) - 2)]
# print the first 3, just so you can see what they look like
print(trigrams[:3])

vocab = set(test_sentence)
word_to_ix = {word: i for i, word in enumerate(vocab)}


class NGramLanguageModeler(nn.Module):

    def __init__(self, vocab_size, embedding_dim, context_size):
        super(NGramLanguageModeler, self).__init__()
        self.embeddings = nn.Embedding(vocab_size, embedding_dim)
        self.linear1 = nn.Linear(context_size * embedding_dim, 128)
        self.linear2 = nn.Linear(128, vocab_size)

    def forward(self, inputs):
        embeds = self.embeddings(inputs).view((1, -1))
        out = F.relu(self.linear1(embeds))
        out = self.linear2(out)
        log_probs = F.log_softmax(out, dim=1)
        return log_probs


losses = []
loss_function = nn.NLLLoss()
model = NGramLanguageModeler(len(vocab), EMBEDDING_DIM, CONTEXT_SIZE)
optimizer = optim.SGD(model.parameters(), lr=0.001)

for epoch in range(10):
    total_loss = torch.Tensor([0])
    for context, target in trigrams:

        # Step 1. Prepare the inputs to be passed to the model (i.e, turn the words
        # into integer indices and wrap them in variables)
        context_idxs = torch.tensor([word_to_ix[w] for w in context], dtype=torch.long)

        # Step 2. Recall that torch *accumulates* gradients. Before passing in a
        # new instance, you need to zero out the gradients from the old
        # instance
        model.zero_grad()

        # Step 3. Run the forward pass, getting log probabilities over next
        # words
        log_probs = model(context_idxs)

        # Step 4. Compute your loss function. (Again, Torch wants the target
        # word wrapped in a variable)
        loss = loss_function(log_probs, torch.tensor([word_to_ix[target]], dtype=torch.long))

        # Step 5. Do the backward pass and update the gradient
        loss.backward()
        optimizer.step()

        # Get the Python number from a 1-element Tensor by calling tensor.item()
        total_loss += loss.item()
    losses.append(total_loss)
print(losses)  # The loss decreased every iteration over the training data!

When I add my own medium sized corpus, the process takes a long time as the example above does not incorporate the concept of mini-batches. So, I decided to try and implement mini-batching into the process.

First, I converted the context ids to a 2d tensor along with the targets:

context_idxs = []
targets = []
for context, target in trigrams:
    # Step 1. Prepare the inputs to be passed to the model (i.e, turn the words
    # into integer indices and wrap them in variables)
    context_idxs.append(torch.Tensor([word_to_ix[w] for w in context]))
    targets.append(torch.Tensor([word_to_ix[target]]))

context_ids = torch.stack(context_idxs, dim=0)
target_ids = torch.stack(targets, dim=0)

Next I tried to run using mini-batches like this:

    current_start = 0
    keep_going = True
    while keep_going:
        if current_start + MINI_BATCH < len(target_ids):
            minibatchids = slice(current_start, current_start + MINI_BATCH -1)
            print(minibatchids)
            current_start = current_start + MINI_BATCH
        else:
            minibatchids = slice(current_start, len(target_ids))
            print(minibatchids)
            keep_going = False

        model.zero_grad()

        # Step 3. Run the forward pass, getting log probabilities over next
        # minibatch of words
        log_probs = model(context_ids[minibatchids])

PyTorch throws the following error:

Traceback (most recent call last):
  File "/mnt/data/projects/PyTorchTutorial/IntroTorch/PyTorch_WordEmbedding_BeigeBook.py", line 102, in <module>
    log_probs = model(context_ids[minibatchids])
  File "/home/david/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/mnt/data/projects/PyTorchTutorial/IntroTorch/PyTorch_WordEmbedding_BeigeBook.py", line 25, in forward
    embeds = self.embeddings(inputs).view((1, -1))
  File "/home/david/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/david/miniconda3/lib/python3.6/site-packages/torch/nn/modules/sparse.py", line 108, in forward
    self.norm_type, self.scale_grad_by_freq, self.sparse)
  File "/home/david/miniconda3/lib/python3.6/site-packages/torch/nn/functional.py", line 1076, in embedding
    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
RuntimeError: Expected tensor for argument #1 'indices' to have scalar type Long; but got CPUFloatTensor instead (while checking arguments for embedding)

I have tried to highlight how my code is changed from the tutorial above. Happy to add more details but not sure what else is helpful.

This leaves me with several questions:

• Is what I am attempting even possible?

If so:

• Am I on the right path?
• Are there any examples of a mini-batch implementation (I can't find any)
• What is the meaning of the error?

Answer 1

You can try retrieving batches of data using following function:

def get_batches(data, batch_size):    
   # iterate through the arrays
   prv = 0
   # Max Sentence Length for padding
   max_len = 128
   for n in range(batch_size, len(data), batch_size):
       # print("Prev={}, N={}".format(prv,n))
       data_x = []
       data_y = []
       for x,y in data[prv:n]:
           data_x.append(torch.Tensor([word_to_ix[w] for w in context], dtype=torch.long))
           data_y.append(torch.Tensor([word_to_ix[target]], dtype=torch.long))

       # pad first seq to desired length
       data_x[0] = nn.ConstantPad1d((0, max_len - data_x[0].shape[0]), 0)(data_x[0])
       data_y[0] = nn.ConstantPad1d((0, max_len - data_y[0].shape[0]), 0)(data_y[0])
    
       # pad all seqs to desired length
       data_x_pad = pad_sequence(data_x, batch_first=True)
       data_y_pad = pad_sequence(data_y, batch_first= True)
    
       # Convert list of tensors to tuple
       data_x_tuple = tuple(data_x_pad)
       data_y_tuple = tuple(data_y_pad)
    
       # Convert tuple of tensors to Stack of Tensors
       data_x = torch.stack(data_x_tuple)
       data_y = torch.stack(data_y_tuple)
    
       prv = n
    
       yield data_x, data_y