0
$\begingroup$

New to python - topic modelling, trying to include bigrams in preprocessing

Had done the following,

def clean(doc):
    punc_free = ''.join([ch for ch in doc.lower() if ch not in exclude])
    stop_free = ' '.join([i for i in punc_free.split() if i not in stop])
    normalized = ' '.join(lemma.lemmatize(word) for word in stop_free.split())
    return normalized
doc_clean = [clean(doc).split() for doc in corpus]
print(doc_clean)

bigrams = []
for word in doc_clean:
    sequence = word_tokenize(doc_clean) 
    bigrams.extend(list(ngrams(sequence, 2)))

And had the following errors


---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-25-bfeeadb225f9> in <module>
      1 bigrams = []
      2 for word in doc_clean:
----> 3     sequence = word_tokenize(doc_clean)
      4     bigrams.extend(list(ngrams(sequence, 2)))

~\Anaconda3\lib\site-packages\nltk\tokenize\__init__.py in word_tokenize(text, language, preserve_line)
    141     :type preserve_line: bool
    142     """
--> 143     sentences = [text] if preserve_line else sent_tokenize(text, language)
    144     return [
    145         token for sent in sentences for token in _treebank_word_tokenizer.tokenize(sent)

~\Anaconda3\lib\site-packages\nltk\tokenize\__init__.py in sent_tokenize(text, language)
    103     """
    104     tokenizer = load('tokenizers/punkt/{0}.pickle'.format(language))
--> 105     return tokenizer.tokenize(text)
    106 
    107 

~\Anaconda3\lib\site-packages\nltk\tokenize\punkt.py in tokenize(self, text, realign_boundaries)
   1267         Given a text, returns a list of the sentences in that text.
   1268         """
-> 1269         return list(self.sentences_from_text(text, realign_boundaries))
   1270 
   1271     def debug_decisions(self, text):

~\Anaconda3\lib\site-packages\nltk\tokenize\punkt.py in sentences_from_text(self, text, realign_boundaries)
   1321         follows the period.
   1322         """
-> 1323         return [text[s:e] for s, e in self.span_tokenize(text, realign_boundaries)]
   1324 
   1325     def _slices_from_text(self, text):

~\Anaconda3\lib\site-packages\nltk\tokenize\punkt.py in <listcomp>(.0)
   1321         follows the period.
   1322         """
-> 1323         return [text[s:e] for s, e in self.span_tokenize(text, realign_boundaries)]
   1324 
   1325     def _slices_from_text(self, text):

~\Anaconda3\lib\site-packages\nltk\tokenize\punkt.py in span_tokenize(self, text, realign_boundaries)
   1311         if realign_boundaries:
   1312             slices = self._realign_boundaries(text, slices)
-> 1313         for sl in slices:
   1314             yield (sl.start, sl.stop)
   1315 

~\Anaconda3\lib\site-packages\nltk\tokenize\punkt.py in _realign_boundaries(self, text, slices)
   1352         """
   1353         realign = 0
-> 1354         for sl1, sl2 in _pair_iter(slices):
   1355             sl1 = slice(sl1.start + realign, sl1.stop)
   1356             if not sl2:

~\Anaconda3\lib\site-packages\nltk\tokenize\punkt.py in _pair_iter(it)
    315     """
    316     it = iter(it)
--> 317     prev = next(it)
    318     for el in it:
    319         yield (prev, el)

~\Anaconda3\lib\site-packages\nltk\tokenize\punkt.py in _slices_from_text(self, text)
   1325     def _slices_from_text(self, text):
   1326         last_break = 0
-> 1327         for match in self._lang_vars.period_context_re().finditer(text):
   1328             context = match.group() + match.group('after_tok')
   1329             if self.text_contains_sentbreak(context):

TypeError: expected string or bytes-like object
$\endgroup$
2
  • $\begingroup$ Could you show what doc_clean looks like? and also if you are iterating through doc_clean, then it should be word_tokenize(word) in your loop. $\endgroup$ – Danny Jul 10 '19 at 12:54
  • $\begingroup$ please provide a minimal code example that reproduces the error $\endgroup$ – oW_ Jul 10 '19 at 15:33

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.