-1
$\begingroup$

I Use the random foraest model for the text classification

below is the code:

df_RB = pd.read_excel(r'C:\expance_Category_sample.xlsx')
df_RB[DESCRIPTION] =  df_RB[DESCRIPTION].str.replace(r'\W', ' ')
# remove all single characters
df_RB[DESCRIPTION] =  df_RB[DESCRIPTION].str.replace(r'\s+[a-zA-Z]\s+', ' ')
# Remove single characters from the start
df_RB[DESCRIPTION] =  df_RB[DESCRIPTION].str.replace(r'\^[a-zA-Z]\s+', ' ')
# Substituting multiple spaces with single space
df_RB[DESCRIPTION] =  df_RB[DESCRIPTION].str.replace(r'\s+', ' ',)

#    Remove the number from the string
df_RB[DESCRIPTION] = df_RB[DESCRIPTION].str.replace('\d+','')
df_RB[DESCRIPTION]  = df_RB[DESCRIPTION].str.lower() 

#get the bag of word
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(max_features=1500, min_df=5, max_df=0.7, stop_words=stopwords.words('english'))
X = vectorizer.fit_transform(df_RB[DESCRIPTION]).toarray()
#get TFIDF
from sklearn.feature_extraction.text import TfidfTransformer
tfidfconverter = TfidfTransformer()
X = tfidfconverter.fit_transform(X).toarray()
y = df_RB['Expance Type']
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

from sklearn.ensemble import RandomForestClassifier

classifier = RandomForestClassifier(n_estimators=10000, random_state=0)

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

from sklearn.ensemble import RandomForestClassifier

classifier = RandomForestClassifier(n_estimators=10000, random_state=0)

classifier.fit(X_train, y_train) 

y_pred = classifier.predict(X_test)

from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

#Test the accuracy
print(confusion_matrix(y_test,y_pred))
print(classification_report(y_test,y_pred))
print(accuracy_score(y_test, y_pred))

pickle.dump(classifier, open('RandomforestLineDescription_model.sav', 'wb'))


loaded_model = pickle.load(open('RandomforestLineDescription_model.sav', 'rb'))

#Now i am trying to get the predicted value on actual dataset 
#below is code .

df_bc[DESCRIPTION] =  df_bc[DESCRIPTION].str.replace(r'\W', ' ')
# remove all single characters
df_bc[DESCRIPTION] =  df_bc[DESCRIPTION].str.replace(r'\s+[a-zA-Z]\s+', ' ')
 # Remove single characters from the start
df_bc[DESCRIPTION] =  df_bc[DESCRIPTION].str.replace(r'\^[a-zA-Z]\s+', ' ')
# Substituting multiple spaces with single space
df_bc[DESCRIPTION] =  df_bc[DESCRIPTION].str.replace(r'\s+', ' ',)

#    Remove the number from the string
df_bc[DESCRIPTION] = df_bc[DESCRIPTION].str.replace('\d+','')
df_bc[DESCRIPTION]  = df_bc[DESCRIPTION].str.lower() 

#df_bc['Labelencodedvalue'] = loaded_model.predict(feature_encoded)

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(max_features=1500,min_df=5, max_df=0.7, stop_words=stopwords.words('english'))
X1 = vectorizer.fit_transform(df[DESCRIPTION]).toarray()

#get TFIDF
from sklearn.feature_extraction.text import TfidfTransformer
tfidfconverter = TfidfTransformer()
X1 = tfidfconverter.fit_transform(X1).toarray()

df_bc['Labelencodedvalue'] = loaded_model.predict(X1)

#it give the error 
ValueError: Number of features of the model must match the input. Model n_features is 994 and input n_features is 471 

#Randomforest model is 
RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
            max_depth=None, max_features='auto', max_leaf_nodes=None,
            min_impurity_decrease=0.0, min_impurity_split=None,
            min_samples_leaf=1, min_samples_split=2,
            min_weight_fraction_leaf=0.0, n_estimators=10000, n_jobs=1,
            oob_score=False, random_state=0, verbose=0, warm_start=False)

Surely i am missing something , if any one can assist it would be grateful

I get the accuracy result is around 97 % which i cross check it was good now when I am trying to use the same model in actual dataset which has around 100k record it is giving the error

Number of features of the model must match the input.

$\endgroup$
  • $\begingroup$ Hi and welcome to the site! Can you please format the code properly using the available formatting features? It is really hard to read currently. $\endgroup$ – Sammy Dec 12 '19 at 10:29
  • $\begingroup$ I hope now it readable format , please let me know if still any challenge to read the code . $\endgroup$ – Neeraj Verma Dec 16 '19 at 5:04
0
$\begingroup$

You should be converting your dataset to CountVec/TFIDF and then create the Train and Test. Else there will be feature mismatch.

$\endgroup$
  • $\begingroup$ Hi Syenix, Thank you for response , i split my sample dataset through below code from sklearn.model_selection import train_test_split X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) After that i get the predicted value for test data till that it is working fine . i save the model and used the same model for actual dataset which has 100k records then i am getting the error . hope it is clear to you ... $\endgroup$ – Neeraj Verma Dec 16 '19 at 6:53

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.