I'm trying to predict housing prices from a Kaggle dataset using an MLP with 3 hidden layers (10 neurons each). Having read about MLPs and backprop in the CS229 notes, I tried to do my own implementation of an MLP trained with backprop (see page 109) and stochastic gradient descent.

However, the model isn't learning - the loss increases to infinity, and Python returns "RuntimeWarning: overflow encountered in matmul". This does not happen if I implement the same architecture in scikit-learn (450 iterations and a 0.01 constant learning rate in both scenarios). I assume that there is some bug in my code that I haven't been able to find yet. I should also mention that I standardized my data in both cases.

Here's my code:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

df = pd.read_csv('housing.csv')

X = df.drop('median_house_value',axis=1)
Y = df['median_house_value']

scaler = StandardScaler()

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=45)

X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)

def sigma(z):
    return np.maximum(0, z)

def sigma_prime(z):
    return np.where(z > 0, 1, 0)

def MSE(z, y):
    return 0.5 * np.dot((z - y), (z - y))

class Layer():
    def __init__(self, previous_neuron_count, neuron_count):
        self.previous_neuron_count = previous_neuron_count
        self.neuron_count = neuron_count        

        self.weights = np.random.randn(self.neuron_count, self.previous_neuron_count)
        self.biases = np.random.randn(1, self.neuron_count)

    def MM(self, previos_layer):
        return ((self.weights @ previos_layer) + self.biases).reshape(self.neuron_count)

class Network():
    def __init__(self, hiden_layers_count: int, hidden_neurons_count: int, output_neurons_count: int, X: np.ndarray, Y: np.ndarray):
        self.hidden_layers_count = hiden_layers_count
        self.X = X
        self.Y = Y
        self.input_layer = X[0]

        self.hidden_neurons_count = hidden_neurons_count
        self.output_neurons_count = output_neurons_count
        self.first_layer = Layer(self.input_layer.shape[0], self.hidden_neurons_count)
        self.layers = np.array([self.first_layer])

        for i in range(1, self.hidden_layers_count):
            self.layers = np.append(self.layers, Layer(self.layers[i - 1].neuron_count, self.layers[i - 1].neuron_count))
        self.layers = np.append(self.layers, Layer(self.layers[-1].neuron_count, self.output_neurons_count))

    def forward_propagation(self):
        self.current_activation = sigma(self.layers[0].MM(self.input_layer))
        self.basic_modules = [[self.layers[0].MM(self.input_layer), self.current_activation]]
        for layer in range(1, len(self.layers)-1):
            self.current_activation = sigma(self.layers[layer].MM(self.current_activation))
            self.basic_modules.append([self.layers[layer].MM(self.current_activation), self.current_activation])
        self.current_activation = self.layers[-1].MM(self.current_activation)
        self.basic_modules.append([self.current_activation, self.current_activation])
        return self.current_activation
    def backward_propagation(self, output, y: np.ndarray, learning_rate: float):
        loss = MSE(output, y)
        self.current_gradient_z = (output - y) * 1
        self.current_gradient_a = 0
        r = self.hidden_layers_count
        for k in range(r-1, -2, -1):
            self.current_weights_gradient = np.outer(self.current_gradient_z, self.basic_modules[k][1])
            self.current_biases_gradient = self.current_gradient_z
            self.layers[k+1].weights -= learning_rate * self.current_weights_gradient
            self.layers[k+1].biases -= learning_rate * self.current_biases_gradient
            if k >= 1:
                self.current_gradient_a = self.layers[k+1].weights.T @ self.current_gradient_z
                self.current_gradient_z = sigma_prime(np.array(self.basic_modules[k][0])) * self.current_gradient_a
    def train(self, learning_rate: float, iterations: int):
        loss_data = np.array([])
        for i in range(iterations):
            sample_index = np.random.randint(0, self.X.shape[0])
            X_sample, Y_sample = self.X[sample_index], self.Y[sample_index]
            self.input_layer = X_sample
            output = self.forward_propagation()
            self.backward_propagation(output, Y_sample, learning_rate)
            loss = MSE(output, Y_sample)
            loss_data = np.append(loss_data, loss)
        return loss_data

What am I doing wrong?


1 Answer 1


The issue with your implementation seems to be in the calculation of the gradient for the activation of the hidden layers in the backward propagation step of your neural network.

def backward_propagation(self, output, y: np.ndarray, learning_rate: float):
    loss = MSE(output, y)
    self.current_gradient_z = (output - y) * 1
    for k in range(self.hidden_layers_count, -1, -1):
        self.current_weights_gradient = np.outer(self.current_gradient_z, self.basic_modules[k][1])
        self.current_biases_gradient = self.current_gradient_z
        self.layers[k].weights -= learning_rate * self.current_weights_gradient
        self.layers[k].biases -= learning_rate * self.current_biases_gradient
        if k > 0:
            self.current_gradient_a = self.layers[k].weights.T @ self.current_gradient_z
            self.current_gradient_z = sigma_prime(self.basic_modules[k][0]) * self.current_gradient_a

Here are the modifications added in the above backward_propagation method:

  • Update the loop range for iterating over the layers from r-1 to 0 instead of r-1 to -2. This is because you want to loop over all layers including the input layer.
  • Correct the calculation of self.current_gradient_a by applying the derivative of the activation function to the gradient for the current layer's activation.
  • Use the correct gradient of the activation function sigma_prime for calculating self.current_gradient_z.

These modifications should address the issue of increasing loss and the overflow encountered in the matrix multiplication.


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.