To implement co-occurence matrix in sucha a way that number of times word1 occured in context of word2 in neighbourhood of given value, lets say 5. There are 100 words and a list with 1000 sentences. So how can i calculate co-occurence matrix of size (100* 100) using python?
from nltk.tokenize import word_tokenize
from itertools import combinations
from collections import Counter
sentences = ['i go to london', 'you do not go to london','but london goes to you']
vocab = set(word_tokenize(' '.join(sentences)))
print('Vocabulary:\n',vocab,'\n')
token_sent_list = [word_tokenize(sen) for sen in sentences]
print('Each sentence in token form:\n',token_sent_list,'\n')
co_occ = {ii:Counter({jj:0 for jj in vocab if jj!=ii}) for ii in vocab}
k=2
for sen in token_sent_list:
for ii in range(len(sen)):
if ii < k:
c = Counter(sen[0:ii+k+1])
del c[sen[ii]]
co_occ[sen[ii]] = co_occ[sen[ii]] + c
elif ii > len(sen)-(k+1):
c = Counter(sen[ii-k::])
del c[sen[ii]]
co_occ[sen[ii]] = co_occ[sen[ii]] + c
else:
c = Counter(sen[ii-k:ii+k+1])
del c[sen[ii]]
co_occ[sen[ii]] = co_occ[sen[ii]] + c
# Having final matrix in dict form lets you convert it to different python data structures
co_occ = {ii:dict(co_occ[ii]) for ii in vocab}
display(co_occ)
Output:
Vocabulary:
{'london', 'but', 'goes', 'i', 'do', 'you', 'go', 'not', 'to'}
Each sentence in token form:
[['i', 'go', 'to', 'london'], ['you', 'do', 'not', 'go', 'to', 'london'], ['but', 'london', 'goes', 'to', 'you']]
{'london': {'go': 2, 'to': 3, 'but': 1, 'goes': 1},
'but': {'london': 1, 'goes': 1},
'goes': {'london': 1, 'but': 1, 'you': 1, 'to': 1},
'i': {'go': 1, 'to': 1},
'do': {'you': 1, 'go': 1, 'not': 1},
'you': {'do': 1, 'not': 1, 'goes': 1, 'to': 1},
'go': {'london': 2, 'i': 1, 'to': 2, 'do': 1, 'not': 1},
'not': {'do': 1, 'you': 1, 'go': 1, 'to': 1},
'to': {'london': 3, 'i': 1, 'go': 2, 'not': 1, 'goes': 1, 'you': 1}}
PS
- Do text preprocessing yourself (remove punctuations, lemmatization, stemming, blahblah)
- Continue the code for any conversion you want. You have the dict, you can convert it to sparse matrix or pandas datframe
-
$\begingroup$ @HarikaM Welcome! you may consider accept/upvote if it helped! Good luck :) $\endgroup$ – Kasra Manshaei Oct 23 '18 at 13:11
you can try this..
import numpy as np
import pandas as pd
ctxs = [
'krayyem like candy crush more then coffe',
'krayyem plays candy crush all days',
'krayyem do not invite his friends to play candy crush',
'krayyem is smart',
]
l_unique = list(set((' '.join(ctxs)).split(' ')))
mat = np.zeros((len(l_unique), len(l_unique)))
nei = []
nei_size = 3
for ctx in ctxs:
words = ctx.split(' ')
for i, _ in enumerate(words):
nei.append(words[i])
if len(nei) > (nei_size * 2) + 1:
nei.pop(0)
pos = int(len(nei) / 2)
for j, _ in enumerate(nei):
if nei[j] in l_unique and words[i] in l_unique:
mat[l_unique.index(nei[j]), l_unique.index(words[i])] += 1
mat = pd.DataFrame(mat)
mat.index = l_unique
mat.columns = l_unique
display(mat)