'''I used the k-means algorithm to clustering set of documents which are textual data only.
The document has 2lack records.
Surprisingly the result for the clustering is
90% of records is storing in 1 cluster, remaining records are going to another clusters.
It's is not a problem if it chooses correct cluster while predicting the data. I thought uneven distribution of clusters might be the issue behind my problem. If thats not the problem, Suggest me how k-means will predict the correct cluster.
Why this is happening ?
Parameters : number of clusters = 5 (used elbow method to know this), random state = 0 or 42 (both used but no use)
#Here is my code showing how i created vectors
from sklearn.feature_extraction.text import TfidfVectorizer
import _pickle as cPickle
def build_tfidf_vect(series,save_model = True ,) :
vectorizer = TfidfVectorizer(stop_words="english")
vectors = vectorizer.fit_transform(series)
print("Shape of tfidf matrix: {}".format(vectors.shape))
if save_model:
data_struct = {'vectors': vectors, 'vectorizer': vectorizer}
with open('data_2l.bin', 'wb') as f:
cPickle.dump(data_struct, f)
return vectorizer, vectors
import pandas as pd
feed = pd.read_csv('2l_data.csv', encoding='latin')
build_tfidf_vect(feed['Column_name'])
This is how i created clusters.
from sklearn.cluster import MiniBatchKMeans
import scipy
def load_tf_idfvectors():
import time
import _pickle as cPickle
with open(r'C:\Users\data_2l.bin', 'rb') as f:
data_struct = cPickle.load(f)
vectors,vectorizer = data_struct['vectors'], data_struct['vectorizer']
return vectorizer,vectors
def dump(cluster_0,cluster_1,cluster_2,cluster_3,cluster_4):
save_model=True
if save_model:
data_struct = {'cluster0': cluster_0, 'cluster1': cluster_1,'cluster2': cluster_2, 'cluster3': cluster_3,'cluster4': cluster_4}
with open(r'C:\Users\totalclusters_2l.bin', 'wb') as f:
cPickle.dump(data_struct, f)
import pandas as pd
import pickle
data=pd.read_csv(r"C:\Users\New_vectors_data_2l.csv", encoding="latin")
tfidf_vectorizer,tfidf=load_tf_idfvectors()
kmeans = MiniBatchKMeans(n_clusters=5, init= 'k-means++',random_state=0).fit(tfidf)
labels = kmeans.fit_predict(tfidf)
X_n=pd.DataFrame(tfidf,columns=['tf-idf'])
labels_n=pd.DataFrame(labels,columns=['cl_n'])
result = pd.concat([data['Column_name'],X_n,labels_n])
pickle.dump(kmeans, open(filename, 'wb'))
results_0=result.loc[result['cl_n'] ==0 , ['tf-idf','cl_n']]
cluster_0_tf = scipy.sparse.vstack(results_0['tf-idf'].tolist())
.
.
.
results_0=results_0.reset_index()
.
.
cluster0_in=results_0['index'].tolist()
.
.
open_file = open(r'C:\Users\cluster0_2l', "wb")
pickle.dump(cluster0_in, open_file)
open_file.close()
.
.
.....
```
