# Can a Neural Network Measure the Random Error in a Linear Series?

I have been trying to develop a neural network to measure the error in a linear series. What I would like the model to do is infer a linear regression line and then measure the mean absolute error around that line.

I have tried a number of neural network model configurations, including recurrent configurations, but the network learns a weak relationship and then overfits. I have also tried L1 and L2 regularization but neither work.

Any thoughts? Thanks!

Below is the code I am using to simulate the data and a fit sample model:

import numpy as np, matplotlib.pyplot as plt

from keras import layers
from keras.models import Sequential
from keras.backend import clear_session

## Simulate the data:

np.random.seed(20190318)

X = np.array(()).reshape(0, 50)

Y = np.array(()).reshape(0, 1)

for _ in range(500):

i = np.random.randint(100, 110) # Intercept.

s = np.random.randint(1, 10) # Slope.

e = np.random.normal(0, 25, 50) # Error.

X_i = np.round(i + (s * np.arange(0, 50)) + e, 2).reshape(1, 50)

Y_i = np.sum(np.abs(e)).reshape(1, 1)

X = np.concatenate((X, X_i), axis = 0)

Y = np.concatenate((Y, Y_i), axis = 0)

## Training and validation data:

split = 400

X_train = X[:split, :-1]
Y_train = Y[:split, -1:]

X_valid = X[split:, :-1]
Y_valid = Y[split:, -1:]

print(X_train.shape)
print(Y_train.shape)
print()
print(X_valid.shape)
print(Y_valid.shape)

## Graph of one of the series:

plt.plot(X_train[0])

## Sample model (takes about a minute to run):

clear_session()

model_fnn = Sequential()
model_fnn.add(layers.Dense(512, activation = 'relu', input_shape = (X_train.shape[1],)))

# Compile model.

model_fnn.compile(optimizer = Adam(lr = 1e-4), loss = 'mse')

# Fit model.

history_fnn = model_fnn.fit(X_train, Y_train, batch_size = 32, epochs = 100, verbose = False,
validation_data = (X_valid, Y_valid))

## Sample model learning curves:

loss_fnn = history_fnn.history['loss']
val_loss_fnn = history_fnn.history['val_loss']
epochs_fnn = range(1, len(loss_fnn) + 1)

plt.plot(epochs_fnn, loss_fnn, 'black', label = 'Training Loss')
plt.plot(epochs_fnn, val_loss_fnn, 'red', label = 'Validation Loss')
plt.title('FNN: Training and Validation Loss')
plt.legend()
plt.show()


UPDATE:

## Predict.

Y_train_fnn = model_fnn.predict(X_train)
Y_valid_fnn = model_fnn.predict(X_valid)

## Evaluate predictions with training data.

plt.scatter(Y_train, Y_train_fnn)
plt.xlabel("Actual")
plt.ylabel("Predicted")

## Evaluate predictions with training data.

plt.scatter(Y_valid, Y_valid_fnn)
plt.xlabel("Actual")
plt.ylabel("Predicted")


This problem is naturally hard. The underlying function that we try to learn is $$\mathbf{X}=i+s+\mathbf{e} \rightarrow Y=\left \| \mathbf{X} - i - s \right \|_1 = \left \| \mathbf{e} \right \|_1=\sum_d|e_d|$$

where $$i$$ and $$s$$ are unknown random variables. For large $$i$$ and $$s$$, $$\left \| \mathbf{e} \right \|_1$$ is naturally hard to recover from $$\mathbf{X}$$. I found a working example (training error almost zero) by setting the intercept $$i$$ and slope $$s$$ to zero!, drastically shrinking the network size to work better with a small sample size (800), and increased the number of epochs to 800, which was crucial. Also, (true value, error) is plotted at the end for training data.

You can work up from this point to see the effect of increasing $$i$$ and $$s$$ on performance.

import numpy as np, matplotlib.pyplot as plt

from keras import layers
from keras.models import Sequential
from keras.backend import clear_session

## Simulate the data:

np.random.seed(20190318)

dimension = 50

X = np.array(()).reshape(0, dimension)

Y = np.array(()).reshape(0, 1)

for _ in range(1000):

i = 0   # np.random.randint(100, 110) # Intercept.

s = 0   # np.random.randint(1, 10) # Slope.

e = np.random.normal(0, 25, dimension) # Error.

X_i = np.round(i + (s * np.arange(0, dimension)) + e, 2).reshape(1, dimension)

Y_i = np.sum(np.abs(e)).reshape(1, 1)

X = np.concatenate((X, X_i), axis = 0)

Y = np.concatenate((Y, Y_i), axis = 0)

## Training and validation data:

split = 800

X_train = X[:split, :-1]
Y_train = Y[:split, -1:]

X_valid = X[split:, :-1]
Y_valid = Y[split:, -1:]

print(X_train.shape)
print(Y_train.shape)
print()
print(X_valid.shape)
print(Y_valid.shape)

## Graph of one of the series:

plt.plot(X_train[0])

## Sample model (takes about a minute to run):

clear_session()

model_fnn = Sequential()
model_fnn.add(layers.Dense(dimension, activation = 'relu', input_shape = (X_train.shape[1],)))

# Compile model.

model_fnn.compile(optimizer = Adam(lr = 1e-4), loss = 'mse')

# Fit model.

history_fnn = model_fnn.fit(X_train, Y_train, batch_size = 32, epochs = 800, verbose = True,
validation_data = (X_valid, Y_valid))

# Sample model learning curves:

loss_fnn = history_fnn.history['loss']
val_loss_fnn = history_fnn.history['val_loss']
epochs_fnn = range(1, len(loss_fnn) + 1)

plt.figure(1)
offset = 5
plt.plot(epochs_fnn[offset:], loss_fnn[offset:], 'black', label = 'Training Loss')
plt.plot(epochs_fnn[offset:], val_loss_fnn[offset:], 'red', label = 'Validation Loss')
plt.title('FNN: Training and Validation Loss')
plt.legend()

## Predict.

plt.figure(2)

Y_train_fnn = model_fnn.predict(X_train)

## Evaluate predictions with training data.
sorted_index = Y_train.argsort(axis=0)
Y_train_sorted = np.reshape(Y_train[sorted_index], (-1, 1))
Y_train_fnn_sorted = np.reshape(Y_train_fnn[sorted_index], (-1, 1))
plt.plot(Y_train_sorted, Y_train_sorted - Y_train_fnn_sorted)
plt.xlabel("Y(true) train")
plt.ylabel("Y(true) - Y(predicted) train")
plt.show()

• Is there a way to make the problem easier to solve? As humans, we know we can fit a regression line and sum the differences. Can a neural network learn to do this? Commented Mar 19, 2019 at 17:30
• Commented Mar 19, 2019 at 17:36
• Working with @esmailian, we solved it by first creating a network to infer predicted values based on a regression line, then a second network to measure the MAE between the predicted and actual values. Of note is that many more observations (20,000) were required because the network struggled to learn the MAE function with 500 or 800 observations. Commented Mar 21, 2019 at 16:05