1
$\begingroup$

Saving Model Part

import pandas as pd     
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
import pickle
import re 
import string
import tensorflow as tf




data_fake=pd.read_csv('Fake.csv')
data_true=pd.read_csv('True.csv')

data_fake["class"]=0
data_true["class"]=1

data_fake.shape, data_true.shape

data_fake_manual_testing=data_fake.tail(10)
for i in range(23480,23470,-1):
    data_fake.drop([i], axis=0,inplace=True)


data_true_manual_testing=data_true.tail(10)
for i in range(21416,21406,-1):
    data_true.drop([i], axis=0,inplace=True)

data_fake.shape, data_true.shape


data_fake_manual_testing["class"]=0
data_true_manual_testing["class"]=1


data_merge=pd.concat([data_fake,data_true],axis=0)


data=data_merge.drop(['title', 'subject', 'date'],axis=1)

data.isnull().sum()

data=data.sample(frac=1)

data.reset_index(inplace=True)
data.drop(['index'],axis=1,inplace=True)

def wordopt(text):
    text=text.lower()
    text=re.sub('\[.*?\]','',text)
    text=re.sub("\\W"," ",text)
    text=re.sub('https?://\S+|www\.\S+','',text)
    text=re.sub('<.*?>+','',text)
    text=re.sub('[%s]' % re.escape(string.punctuation),'',text)
    text=re.sub('\n','',text)
    text=re.sub('\w*\d\w','',text)
    return text


data['text']=data['text'].apply(wordopt)

x=data['text']
y=data['class']

x_train, x_test, y_train, y_test = train_test_split(x,y,test_size=0.25)
from sklearn.feature_extraction.text import TfidfVectorizer

vectorization = TfidfVectorizer()
xv_train = vectorization.fit_transform(x_train)
xv_test=vectorization.transform(x_test)

pickle.dump(vectorization, open('vectorization.sav', 'wb'))

from sklearn.linear_model import LogisticRegression

LR=LogisticRegression()

LR.fit(xv_train,y_train)
pickle.dump(LR, open('lrModel.sav', 'wb'))

pred_lr=LR.predict(xv_test)
print(classification_report(y_test,pred_lr))


from sklearn.tree import DecisionTreeClassifier


DT=DecisionTreeClassifier()
DT.fit(xv_train,y_train)
pickle.dump(DT, open('dtModel.sav', 'wb'))
pred_dt=DT.predict(xv_test)

print(DT.score(xv_test,y_test))

print(classification_report(y_test,pred_dt))

from sklearn.ensemble import GradientBoostingClassifier

GB=GradientBoostingClassifier(random_state=0)
GB.fit(xv_train,y_train)
pickle.dump(GB, open('gbModel.sav', 'wb'))
predict_gb=GB.predict(xv_test)

print(GB.score(xv_test,y_test))

print(classification_report(y_test,predict_gb))


from sklearn.ensemble import RandomForestClassifier

RF=RandomForestClassifier(random_state=0)
RF.fit(xv_train,y_train)
pickle.dump(RF, open('rfModel.sav', 'wb'))
predict_rf=RF.predict(xv_test)

print(RF.score(xv_test,y_test))

print(classification_report(y_test,predict_rf))

def output_label(n):
    if n==0:
        return 'Fake News'
    elif n==1:
        return 'Not a fake news'
    
def manual_testing(news):
    testing_news={'text':[news]}
    new_def_test=pd.DataFrame(testing_news)
    new_def_test["text"]=new_def_test["text"].apply(wordopt)
    new_x_test=new_def_test["text"]
    new_xv_test=vectorization.transform(new_x_test)
    print(new_xv_test)
    pred_LR=LR.predict(new_xv_test)
    pred_DT=DT.predict(new_xv_test)
    pred_GBC=GB.predict(new_xv_test)
    pred_RFC=RF.predict(new_xv_test)

    return print('\n\n Prediction: {} \nLR prediction:{}\nDT prediction:{}\nGB prediction:{}\nRF prediction:{}'.format(news,output_label(pred_LR[0]),
                                                                                                                       output_label(pred_DT[0]),
                                                                                                                       output_label(pred_GBC[0]),
                                                                                                                       output_label(pred_RFC[0])
                                                                                                                       ))
while True:
    news=str(input('Enter the new:'))
    manual_testing(news)

Loading Model part

from sklearn.feature_extraction.text import TfidfVectorizer
import string
import pandas as pd   
from sklearn.model_selection import train_test_split
import pickle
import re

LR = pickle.load(open('lrModel.sav', 'rb'))
DT = pickle.load(open('dtModel.sav', 'rb'))
GB = pickle.load(open('gbModel.sav', 'rb'))
RF = pickle.load(open('rfModel.sav', 'rb'))



def wordopt(text):
    text = text.lower()
    text = re.sub('\[.*?\]', '', text)
    text = re.sub("\\W", " ", text)
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)
    return text

def output_label(n):
    if n == 0:
        return 'Fake News'
    elif n == 1:
        return 'Not a fake news'

vectorization = TfidfVectorizer()
data_fake = pd.read_csv('Fake.csv')
data_true = pd.read_csv('True.csv')

data_fake["class"] = 0
data_true["class"] = 1

data_fake.shape, data_true.shape

data_fake_manual_testing=data_fake.tail(10)
for i in range(23480,23470,-1):
    data_fake.drop([i], axis=0,inplace=True)


data_true_manual_testing=data_true.tail(10)
for i in range(21416,21406,-1):
    data_true.drop([i], axis=0,inplace=True)

data_fake.shape, data_true.shape

data_fake_manual_testing = data_fake.tail(10)
data_true_manual_testing = data_true.tail(10)

data_fake_manual_testing["class"] = 0
data_true_manual_testing["class"] = 1

data_merge = pd.concat([data_fake, data_true], axis=0)

data = data_merge.drop(['title', 'subject', 'date'], axis=1)

data.isnull().sum()

data = data.sample(frac=1)

data.reset_index(inplace=True)
data.drop(['index'], axis=1, inplace=True)

data['text'] = data['text'].apply(wordopt)

x = data['text']
y = data['class']

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25)
vectorization = TfidfVectorizer()
xv_train = vectorization.fit_transform(x_train)
xv_test = vectorization.transform(x_test)


while True:
    news = str(input('Enter the news:'))
    testing_news={'text':[news]}
    new_def_test=pd.DataFrame(testing_news)
    new_def_test["text"]=new_def_test["text"].apply(wordopt)
    new_x_test=new_def_test["text"]
    new_xv_test=vectorization.transform(new_x_test)
    print(new_xv_test)
    pred_LR = LR.predict(new_xv_test)
    pred_DT = DT.predict(new_xv_test)
    pred_GBC = GB.predict(new_xv_test)
    pred_RFC = RF.predict(new_xv_test)

   
    print('\n\nPrediction: {}\nLR prediction: {}\nDT prediction: {}\nGB prediction: {}\nRF prediction: {}'.format(
        news,
        output_label(pred_LR[0]),
        output_label(pred_DT[0]),
        output_label(pred_GBC[0]),
        output_label(pred_RFC[0])
    ))

This is a basic fake-news-detection machine learning model. Here is the problem occurs when I load the modal but not in the saving part. When I save the model and give some input to predict it works exactly how I want. But when I loaded the model to reuse, I always get that error. Error occurs when input the data to predict

Exception has occurred: ValueError
X has 94630 features, but LogisticRegression is expecting 99780 features as input.
  File "C:\Users\****\****\Desktop\intro_security_test\testAllModels.py", line 93, in <module>
    pred_LR = LR.predict(new_xv_test)
              ^^^^^^^^^^^^^^^^^^^^^^^
ValueError: X has 94630 features, but LogisticRegression is expecting 99780 features as input.
$\endgroup$

1 Answer 1

1
$\begingroup$

You should apply exactly the same steps when doing inference as when you were training model, which includes applying the same vectorizer. In this case you are saving the vectorizer but not loading and applying it when doing inference, instead creating a complete new instance of the TfidfVectorizer and training that again.

$\endgroup$
1
  • $\begingroup$ The vectorizer is an empty object, but xv_train. Can I save xv_train and reuse it as a fitted vectorization? $\endgroup$ Jun 10, 2023 at 9:48

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.