Firstly, in all the linear separator algorithms such as linear regression, logistic regression and the perceptron, adding the bias is as simple as adding a feature column consisting of all 1's. Then the third weight that will be trained will act as the bias $b$.
I have some working code for a multi-class perceptron
First let's generate some artificial data with 2 features. Each distribution of points is going to be Gaussian with a given mean and variance described as a list of lists for each dimension under the variable params.
def gen_data(params, n):
dims = len(params[0])
num_classes = len(params)
x = np.zeros((n*num_classes, dims))
y = np.zeros((n*num_classes,))
for ix, i in enumerate(range(num_classes)):
inst = np.random.randn(n, dims)
for dim in range(dims): x[ix*n:(ix+1)*n,dim] = np.random.normal(params[ix][dim][0],
params[ix][dim][1], n)
y[ix*n:(ix+1)*n] = ix
return x, y
params = [[[ 5,1], [ 5,1]],
[[ 0,1], [ 0,1]],
[[2, 1], [ 2,1]],
[[-2, 1], [ 2,1]]]
n = 300
x, y = gen_data(params, 300)
plt.scatter(x[:,0], x[:,1])
plt.show()
Alright so now we have 4 distributions with different labels. Let's split the data for sanctity's sake.
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.33)
Let's train the weights using the training data
def get_weights(x, y, n_epochs, verbose = 0):
# Append a ones column to the feature for the bias
data = np.ones((x.shape[0], x.shape[1]+1))
data[:, 0:x.shape[1]] = x
# Set the targets as integers for comparison
targets = y.astype(int)
# Initialize the weights as a matrix
# number of classes by number of features
weights = np.ones((len(set(y)), x.shape[1]+1))
for epoch in range(n_epochs):
for i, target in zip(data, targets):
temp = np.dot(i, weights.T)
pred = np.argmax(temp)
# If wrongly predicted update prediction
if pred != target:
weights[target, :] = weights[target, :] + i
weights[pred, :] = weights[pred, :] - i
if verbose == 1:
print('Iteration: ', epoch)
print(weights)
print('---------------------------------------------')
return weights
weights = get_weights(x_train, y_train, n_epochs = 30, verbose = 1)
This converges to approximately this
[[ 23.62752045 16.03867499 -111. ]
[ -3.96545848 -8.66924406 47. ]
[ -0.94290763 -1.84413793 33. ]
[ -14.71915434 -1.52529301 35. ]]
We get an accuracy calculated using the score
def predict(x, weights):
data = np.ones(( x.shape[0], x.shape[1]+1 ))
data[:, 0:x.shape[1]] = x
predictions = np.argmax(np.dot(data, weights.T), axis = 1)
return predictions
def score(x, y, weights):
pred = predict(x, weights)
return sum(pred == y_test)/len(pred)
score(x_test, y_test, weights)
0.8686868686868687
We can check our results using a confusion matrix. For the training set
from sklearn.metrics import confusion_matrix
predictions = predict(x_train, weights)
plt.imshow(confusion_matrix(y_train, predictions))
plt.show()
And the testing set
predictions = predict(x_test, weights)
plt.imshow(confusion_matrix(y_test, predictions))
plt.show()
So we see that in fact our algorithm is performing quite well.
We can then plot our points to see how it is classifying them. I will plot the training points as small circles, and the testing points as larger ones. The dark points are those which are misclassified
colors = ['y', 'r', 'b', 'g', 'k']
# Predict training set
predictions = predict(x_train, weights)
for i, t, p in zip(x_train, y_train, predictions):
if t == p: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 0.2, s=20)
else: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 1)
# Predict test set
predictions = predict(x_test, weights)
for i, t, p in zip(x_test, y_test, predictions):
if t == p: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 0.2)
else: plt.scatter(i[0], i[1], c=colors[int(t)], alpha = 1)
# Plot the linear separators
x1 = np.linspace(np.min(x[:,0]),np.max(x[:,1]),2)
x2 = np.zeros((weights.shape[0], 2))
for ix_w, weight in enumerate(weights):
x2 = 1 * ( - weight[2] - weight[0]*x1) / weight[1]
plt.plot(x1, x2, c = colors[ix_w])
plt.xlabel('Feature 1')
plt.ylabel('Feature 2')
plt.xlim([np.min(x[:,0]), np.max(x[:,0])])
plt.ylim([np.min(x[:,1]), np.max(x[:,1])])
plt.show()
This code generalizes to the binary classification task as well
params = [[[ 5,1], [ 5,1]],
[[ 0,1], [ 0,1]]]
Stop training on convergence
If you want to stop the algorithm based on convergence you can use a stop criteria. For example you can stop training once every weight in your matrix changes by less than a very small number. The very small number we usually choose is machine epsilon 2.220446049250313e-16, which is essentially zero. Sometimes this requirement is too stringent so it can be replaced by any number of significant values.
Change the get_weights code to include the break criteria as
from copy import deepcopy
def get_weights(x, y, n_epochs, verbose = 0):
# Append a ones column to the feature for the bias
data = np.ones((x.shape[0], x.shape[1]+1))
data[:, 0:x.shape[1]] = x
# Set the targets as integers for comparison
targets = y.astype(int)
# Initialize the weights as a matrix
# number of classes by number of features
weights = np.zeros((len(set(y)), x.shape[1]+1))
past_weights = np.zeros((len(set(y)), x.shape[1]+1))
for epoch in range(n_epochs):
for i, target in zip(data, targets):
temp = np.dot(i, weights.T)
pred = np.argmax(temp)
# If wrongly predicted update prediction
if pred != target:
weights[target, :] = weights[target, :] + i
weights[pred, :] = weights[pred, :] - i
if np.abs(weights - past_weights).all() < np.finfo(float).eps:
break
past_weights = deepcopy(weights)
if verbose == 1:
print('Iteration: ', epoch)
print(weights)
print('---------------------------------------------')
return weights
trainthe entirety of your training set? What are their respective labels? $\endgroup$ – JahKnows May 28 '18 at 3:32data_testvariable? $\endgroup$ – JahKnows May 28 '18 at 3:33