I'm working a multi-class text classification project.
After splitting the dataset into train and test datasets, I've applied the below function on the train dataset (AKA pre processing):
STOPWORDS = set(stopwords.words('english'))
def clean_text(text):
# lowercase text
text = text.lower()
# delete bad symbols
text = re.sub(r"(@\[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)|^rt|http.+?", "", text)
# delete stopwords from text
text = ' '.join(word for word in text.split() if word not in STOPWORDS)
# Stemming the words
text = ' '.join([stemmer.stem(word) for word in text.split()])
return text
To my surprise, I've got much worst results (i.e. va_accuracy) applying on the train dataset rather than just "do nothing" (59% vs 69%)
I've literally commented out the apply line in the below section:
all_data = dataset.sample(frac=1).reset_index(drop=True)
train_df, valid = train_test_split(all_data, test_size=0.2)
train_df['text'] = train_df['text'].apply(clean_text)
What am I missing? How can it be that pre processing steps decreased accuracy?
A bit more info
I forgot to mention I'm using the below to tokenize the text:
X_train = train.iloc[:, :-1]
y_train = train.iloc[:, -1:]
X_test = valid.iloc[:, :-1]
y_test = valid.iloc[:, -1:]
weights = class_weight.compute_class_weight(class_weight='balanced', classes=np.unique(y_train),
y=y_train.values.reshape(-1))
le = LabelEncoder()
le.fit(weights)
class_weights_dict = dict(zip(le.transform(list(le.classes_)), weights))
tokenizer = Tokenizer(num_words=vocab_size, oov_token='<OOV>')
tokenizer.fit_on_texts(X_train['text'])
train_seq = tokenizer.texts_to_sequences(X_train['text'])
train_padded = pad_sequences(train_seq, maxlen=max_length, padding=padding_type, truncating=trunc_type)
validation_seq = tokenizer.texts_to_sequences(X_test['text'])
validation_padded = pad_sequences(validation_seq, maxlen=max_length, padding=padding_type, truncating=trunc_type)
Later on I'm fitting all into the model as follows:
model = Sequential()
model.add(Embedding(vocab_size, embedding_dim, input_length=train_padded.shape[1]))
model.add(Conv1D(48, len(GROUPS), activation='relu', padding='valid'))
model.add(GlobalMaxPooling1D())
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(len(GROUPS), activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
epochs = 100
batch_size = 32
history = model.fit(train_padded, training_labels, shuffle=True ,
epochs=epochs, batch_size=batch_size,
class_weight=class_weights_dict,
validation_data=(validation_padded, validation_labels),
callbacks=[ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=0.0001),
EarlyStopping(monitor='val_loss', mode='min', patience=2, verbose=1),
EarlyStopping(monitor='val_accuracy', mode='max', patience=5, verbose=1)])