As a newbie in 'pytorch', I am building a neural network for classification of faulty water pumps in Tazania for this competition I am also using ax-platform for hyperparameter tuning.
Yes methods such as gradient boosting classifiers and random forest probably works well or even better than neural network classifier for this tabular data problem but I want to practice using pytorch.
The problem is that when I am doing optimise from ax-platform I am getting accuracy score not more than 54% and I wish to improve that. It jumps around certain number only as shown here:
I tried to do some debugging in my evaluate function by printing out predicted and labels
def evaluate(net, testloader):
correct = 0
total = 0
with torch.no_grad():
for data in testloader:
inputs, labels = map(Variable,data)
outputs = net(inputs)
_,predicted = torch.max(outputs,1)
print("row in testloader")
print("predicted: ")
print(predicted)
print("labels: ")
print(labels)
total += labels.size(0)
correct += (predicted == labels).sum().item()
# print(str(correct)+"/"+str(total))
print('Accuracy of the network: %d %%' % (
100 * correct / total))
return 100 * correct / total
and net(inputs) is giving a tensor of all ones. It should have been a tensors with varied classifications/numbers of [0,1,2] .
I have tried (maybe not well):
- using SGD and ADAM
- 1,2 and 3 hidden layers
- different epoches
- 'Dropout' with different dropout probabilites
- disabling biases
- having different learning rates
- different number of neurons
- different total trails
- different betas and eps for ADAM algorithm
- 'log_scale = True' in parameters optimsation
sci-kit learnStandardScalerontrain_valuesandtest_valuesBut none broke the barrier.
How can I break the 54% accuracy barrier here? What are the gaps in my knowledges? What bugs in the code that causes this?
Here is the code:
import numpy as np
import pandas as pd
import torch
import torchvision
import torchvision.transforms as transforms
from ax import optimize
from ax.plot.contour import plot_contour
from ax.plot.trace import optimization_trace_single_method
from ax.utils.notebook.plotting import render
from sklearn import preprocessing
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, OrdinalEncoder, StandardScaler
from torch import functional, nn, optim
from torch.autograd import Variable
import math
rng =0
ordinalEncorder = OrdinalEncoder()
labelEncoder = LabelEncoder()
standardScaler = StandardScaler()
d = {
"funder": "string","installer": "string",
"wpt_name": "string", "basin": "string",
"subvillage": "string", "region": "string", "lga": "string", "ward": "string",
"public_meeting": "string", "recorded_by": "string", "scheme_management": "string",
"scheme_name": "string", "permit": "string",
"extraction_type": "string", "extraction_type_group": "string", "extraction_type_class": "string",
"management": "string",
"management_group": "string", "payment": "string",
"payment_type": "string",
"water_quality": "string", "quality_group": "string",
"quantity": "string", "quantity_group": "string", "source": "string",
"source_type": "string",
"source_class": "string", "waterpoint_type": "string",
"waterpoint_type_group": "string",
"date_recorded": "string",
}
str_cat= [
"funder","installer",
"wpt_name" , "basin",
"subvillage", "region",
"lga", "ward",
"public_meeting", "recorded_by",
"scheme_management", "scheme_name",
"permit", "extraction_type",
"extraction_type_group",
"extraction_type_class", "management",
"management_group",
"payment", "payment_type", "water_quality", "quality_group",
"quantity",
"quantity_group", "source",
"source_type", "source_class",
"waterpoint_type",
"waterpoint_type_group",
"date_recorded",
]
test_values =pd.read_csv("./test-set-values.csv")
train_values = pd.read_csv("./training-set-values.csv")
train_labels = pd.read_csv("./training-set-labels.csv")
length = train_values.shape[0]
train_batch_size = 100
test_batch_size= 100
train_labels['status_group'] = labelEncoder.fit_transform(train_labels.status_group.astype('string'))
train_values[str_cat] = ordinalEncorder.fit_transform(train_values[str_cat].astype(d).fillna(""))
test_values[str_cat] = ordinalEncorder.fit_transform(test_values[str_cat].astype(d).fillna(""))
train_values[str_cat] = standardScaler.fit_transform(train_values[str_cat])
test_values[str_cat]=standardScaler.fit_transform(test_values[str_cat])
train_values = train_values.astype('float32').values
test_values = test_values.astype('float32').values
train_labels = train_labels.status_group.astype('long').values
x_train,x_test,y_train,y_test = map(torch.from_numpy,
train_test_split(train_values,train_labels,test_size=0.2,
random_state=rng,shuffle=True))
train = torch.utils.data.TensorDataset(x_train,y_train)
test = torch.utils.data.TensorDataset(x_test,y_test)
train_loader = torch.utils.data.DataLoader(train,batch_size=train_batch_size,shuffle = False)
test_loader = torch.utils.data.DataLoader(test,batch_size=test_batch_size,shuffle = False)
class Net(nn.Module):
def __init__(self,hidden1,hidden2,dropoutProbabilities1):
super(Net,self).__init__()
self.layer1 = nn.Linear(40,hidden1,)
self.layer2 = nn.Linear(hidden1,hidden2)
self.layer3 = nn.Linear(hidden2,3)
# self.layer2 = nn.Linear(hidden1,3) # layer 2 configuration if layer 3 is not used.
self.layer4 = nn.Linear(3,3,)
self.activation =nn.ReLU()
self.dropout1 = nn.Dropout(p=dropoutProbabilities1)
# self.dropout2 = nn.Dropout(p=dropoutProbabilities2)
# self.batchNormalisation1 = nn.BatchNorm1d(hidden1)
# self.batchNormalisation2 = nn.BatchNorm1d(hidden2)
def forward(self, x):
x = self.layer1(x)
# x = self.batchNormalisation1(x)
x = self.layer2(x)
x = self.activation(x)
x = self.dropout1(x)
# x = self.batchNormalisation2(x)
x = self.layer3(x)
x = self.activation(x)
x = self.dropout1(x)
x = self.layer4(x)
return x
def train(net, parameterization, trainloader):
optimizer = optim.Adam(net.parameters(),
lr=parameterization['lr'],
weight_decay=parameterization['weight_decay'],
maximize = True
)
criterion = nn.CrossEntropyLoss()
for epoch in range(5):
for data in trainloader:
inputs, labels = map(Variable,data)
optimizer.zero_grad()
loss = criterion(net(inputs), labels)
loss.backward()
optimizer.step()
return net
def evaluate(net, testloader):
correct = 0
total = 0
with torch.no_grad():
for data in testloader:
inputs, labels = map(Variable,data)
outputs = net(inputs)
_,predicted = torch.max(outputs,1)
# print("row in testloader")
# print("predicted: ")
# print(predicted)
# print("labels: ")
# print(labels)
total += labels.size(0)
correct += (predicted == labels).sum().item()
# print(str(correct)+"/"+str(total))
print('Accuracy of the network: %d %%' % (
100 * correct / total))
return 100 * correct / total
def train_evaluate(parameters):
net = Net(parameters["hidden1"],parameters['hidden2'],parameters['dropoutProbabilities1'])
# net = Net(parameters["hidden1"],parameters['hidden2'])
net = train(net, parameters, train_loader)
return evaluate(net, test_loader)
parameters=[
{"name": "lr", "type": "range",
"value_type":'float', "bounds": [0.001, 0.5],'log_scale':True},
{"name": "weight_decay", "type": "range",
"value_type":'float',"bounds": [0.1, 0.9999],'log_scale':True},
{"name": "hidden1", "type":"range",
"value_type":'int',"bounds":[5,1100],'log_scale':True},
{"name": "hidden2", "type":"range",
"value_type":'int',"bounds":[5,1100],'log_scale':True},
{"name":"dropoutProbabilities1","type":"range","value_type"
:"float","bounds":[0.1,0.4]},
]
best_parameters, values, experiment, model = optimize(
parameters= parameters,
evaluation_function=train_evaluate,
objective_name='accuracy',
minimize=False,
total_trials=10
)