Return to Question

another function like

for s in sentences :                 
             tokenized_s = tokenizer.tokenize(s)                
             indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_s)
             tokens_tensor = torch.tensor([indexed_tokens]).to(device)
             with torch.no_grad():
                encoded_layers, _ = model(tokens_tensor)
             bert_embedding = encoded_layers[11].squeeze(0)
             split_s = s.split()
             tokens_embedding = []
             j = 0
             for full_token in split_s:
                 curr_token = ''
                 x = 0
                 for i,_ in enumerate(tokenized_s[1:]): # disregard CLS
                     token = tokenized_cap[i+j]
                     piece_embedding = bert_embedding[i+j]
                        
                        # full token
                     if token == full_token and curr_token == '' :
                         tokens_embedding.append(piece_embedding)
                         j += 1
                         break
                     else: # partial token
                         x += 1
                         if curr_token == '':
                             tokens_embedding.append(piece_embedding)
                             curr_token += token.replace('#', '')
                         else:
                             tokens_embedding[-1] = torch.add(tokens_embedding[-1], piece_embedding)
                             curr_token += token.replace('#', '')
                             if curr_token == full_token: # end of partial
                                j += x
                                break                            

             s_embedding = torch.stack(tokens_embedding) 
             embeddings.append(s_embedding)
    embedding_matrix = torch.stack(embeddings)

I'm confused , does the second function for word-embedding or sub-word too.

i tried to use the second code in image captioning task but got result like

CLS r o o m l i v i n g [ S E P ] [ S E P ] [ S E P ] [ S E P ]
[ S E P ] [ S E P ] [ S E P ] s e p s e p s e p s e p s e p 
s e p s e p s e p s e p s e p s e p s e p s e p s e p s e p
s e p s e p s e p s e p s e p s e p s e p s e p s e p s e p 
s e p s e p s e p s e p s e p s e p s e p s e p s e p s e p 
s e p s e p s e p s e p s e p 

i got words not full sentences . i think i missed something too . i tried to pass the results from the function using

inputs2 = Input(shape=(max_length,))
   
sent = Embedding(vocab_size, 3072, mask_zero=True)(inputs2)
    
lstm3 = LSTM(512)(sent)

model.layers[1].set_weights([embedding_matrix])

model.layers[1].trainable = False

I'm confused , does the second function for word-embedding or sub-word too

I'm confused , does the second function for word-embedding or sub-word too.

i tried to use the second code in image captioning task but got result like

CLS r o o m l i v i n g [ S E P ] [ S E P ] [ S E P ] [ S E P ]
[ S E P ] [ S E P ] [ S E P ] s e p s e p s e p s e p s e p 
s e p s e p s e p s e p s e p s e p s e p s e p s e p s e p
s e p s e p s e p s e p s e p s e p s e p s e p s e p s e p 
s e p s e p s e p s e p s e p s e p s e p s e p s e p s e p 
s e p s e p s e p s e p s e p 

i got words not full sentences . i think i missed something too . i tried to pass the results from the function using

inputs2 = Input(shape=(max_length,))
   
sent = Embedding(vocab_size, 3072, mask_zero=True)(inputs2)
    
lstm3 = LSTM(512)(sent)

model.layers[1].set_weights([embedding_matrix])

model.layers[1].trainable = False

for cap_idx in sentences :                 
        while len(cap_idx) < max_len:
             cap_idx.append(PAD)
             cap = ' '.join([vocab.idx2word[word_idx.item()] for word_idx in cap_idx])
             cap = u'[CLS] '+cap
             tokenized_cap = tokenizer.tokenize(cap)                
             indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_cap)
             tokens_tensor = torch.tensor([indexed_tokens]).to(device)
             with torch.no_grad():
                encoded_layers, _ = model(tokens_tensor)
             bert_embedding = encoded_layers[11].squeeze(0)
             split_cap = cap.split()
             tokens_embedding = []
             j = 0
             for full_token in split_cap:
                 curr_token = ''
                 x = 0
                 for i,_ in enumerate(tokenized_cap[1:]): # disregard CLS
                     token = tokenized_cap[i+j]
                     piece_embedding = bert_embedding[i+j]
                        
                        # full token
                     if token == full_token and curr_token == '' :
                         tokens_embedding.append(piece_embedding)
                         j += 1
                         break
                     else: # partial token
                         x += 1
                         if curr_token == '':
                             tokens_embedding.append(piece_embedding)
                             curr_token += token.replace('#', '')
                         else:
                             tokens_embedding[-1] = torch.add(tokens_embedding[-1], piece_embedding)
                             curr_token += token.replace('#', '')
                             if curr_token == full_token: # end of partial
                                j += x
                                break                            

             cap_embedding = torch.stack(tokens_embedding) 
             embeddings.append(cap_embedding)
    embedding_matrix = torch.stack(embeddings)

for s in sentences :                 
             tokenized_s = tokenizer.tokenize(s)                
             indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_s)
             tokens_tensor = torch.tensor([indexed_tokens]).to(device)
             with torch.no_grad():
                encoded_layers, _ = model(tokens_tensor)
             bert_embedding = encoded_layers[11].squeeze(0)
             split_s = s.split()
             tokens_embedding = []
             j = 0
             for full_token in split_s:
                 curr_token = ''
                 x = 0
                 for i,_ in enumerate(tokenized_s[1:]): # disregard CLS
                     token = tokenized_cap[i+j]
                     piece_embedding = bert_embedding[i+j]
                        
                        # full token
                     if token == full_token and curr_token == '' :
                         tokens_embedding.append(piece_embedding)
                         j += 1
                         break
                     else: # partial token
                         x += 1
                         if curr_token == '':
                             tokens_embedding.append(piece_embedding)
                             curr_token += token.replace('#', '')
                         else:
                             tokens_embedding[-1] = torch.add(tokens_embedding[-1], piece_embedding)
                             curr_token += token.replace('#', '')
                             if curr_token == full_token: # end of partial
                                j += x
                                break                            

             s_embedding = torch.stack(tokens_embedding) 
             embeddings.append(s_embedding)
    embedding_matrix = torch.stack(embeddings)