Text Mining on Large Dataset

Question

I have a large data set(460 Mb) which has a column - Log with 386551 rows. I wish to use clustering and N-Gram approach to form word cloud. My code is as follows:

library(readr)
AMC <- read_csv("All Tickets.csv")
Desc <- AMC[,4]

#Very large data hence breaking it down for creating corpus
#DataframeSource has been used insted of VectorSource is to be able to       handle the data

library(tm)
docs_new <- data.frame(Desc)

test1 <- docs_new[1:100000,]
test2 <- docs_new[100001:200000,]
test3 <- docs_new[200001:300000,]
test4 <- docs_new[300001:386551,]
test1 <- data.frame(test1)
test1 <- Corpus(DataframeSource(test1))
test2 <- data.frame(test2)
test2 <- Corpus(DataframeSource(test2))
test3 <- data.frame(test3)
test3 <- Corpus(DataframeSource(test3))
test4 <- data.frame(test4)
test4 <- Corpus(DataframeSource(test4))

# attach all the corpus
docs_new <- c(test1,test2,test3,test4)

docs_new <- tm_map(docs_new, tolower)
docs_new <- tm_map(docs_new, removePunctuation)
docs_new <- tm_map(docs_new, removeNumbers)
docs_new <- tm_map(docs_new, removeWords, stopwords(kind = "en"))
docs_new <- tm_map(docs_new, stripWhitespace)
docs_new <- tm_map(docs_new, stemDocument)
docs_new <- tm_map(docs_new, PlainTextDocument)

#tokenizer for tdm with ngrams
library(RWeka)
options(mc.cores=1) 
BigramTokenizer <- function(x) NGramTokenizer(x, Weka_control(min = 2, max =2))
tdm <- TermDocumentMatrix(docs_new, control = list(tokenize = BigramTokenizer))

This is giving me results as follows:

TermDocumentMatrix (terms: 1874071, documents: 386551)>>
Non-/sparse entries: 17313767/724406705354
Sparsity           : 100%
Maximal term length: 733
Weighting          : term frequency (tf)

I then converted it to dgMatrix using following :

library("Matrix")
mat <- sparseMatrix(i=tdm$i, j=tdm$j, x=tdm$v, dims=c(tdm$nrow, tdm$ncol))

While trying to use the following I am getting memory size error:

removeSparseTerms(tdm, 0.2)

Please suggest further as I am new to Text Analytics.

Answer 1

You are using R, and everything that you are currently working on will be held in memory hence the error. You just don't have enough of it.

You might be better off creating the frequencies of the terms from the original splits as opposed to creating one big file.

Then after you have the frequencies adding them all together.

Personally I use this code to create my wordclouds.

##Clean code for single column of a dataframe, in this case named Text
  x = alltweets
  tweets.text <- x$text
  tweets.text.cleaned <- gsub("@\\w+ *#", "", tweets.text)
  tweets.text.cleaned <- gsub("(f|ht)tp(s?)://(.*)[.][a-z]+", "", tweets.text.cleaned)
  tweets.text.cleaned <- gsub("[^0-9A-Za-z///' ]", "", tweets.text.cleaned)
  tweets.text.corpus <- Corpus(VectorSource(tweets.text.cleaned))
  tweets.text.final <- tm_map (tweets.text.corpus, removePunctuation, mc.cores=1)
  tweets.text.final2 <- tm_map (tweets.text.final, content_transformer(tolower), mc.cores=1)
  tweets.text.final2 <- tm_map(tweets.text.final2, removeNumbers, mc.cores=1)
  tweets.text.final2 <- tm_map(tweets.text.final2, removePunctuation, mc.cores=1)
  tweets.text.final2 <- tm_map(tweets.text.final2,removeWords, stopwords("English"), mc.cores=1)
  tweets.text.final2 = tm_map(tweets.text.final2, removeWords, c("amp", "&"))
#create corpus
   housing.tweets.corpus <- Corpus(VectorSource(tweets.text))

#clean up by removing stop words
   housing.tweets.corpus <- tm_map(housing.tweets.corpus, function(x)removeWords(x,stopwords()))

#install wordcloud if not already installed
   install.packages("wordcloud")
   library("wordcloud")

#generate wordcloud
   wordcloud(housing.tweets.corpus,min.freq = 2, scale=c(7,0.5),colors=brewer.pal(8, "Dark2"),  random.color= TRUE, random.order = FALSE, max.words = 500)