Tensorflow regression predicting 1 for all inputs

Question

I'm trying to predict a UPDRS score (regression) from the parkinsons dataset. I wanted to start with a fairly basic structure and the add in more layers/techniques to improve performance but no matter what I do the network constantly predicts 1 for every input.

I've been banging my head against this problem for a couple of days now with no success. Originally I was using RELU and thought it was a case of dead neurons but using leaky RELU and monitoring the outputs I can see them firing.

Changing the network structure or training steps always results with the same horrible MSE of 896.39105. This leads me to believe the mistake is somewhere in the normalization but when I check the intermediate values they all seem fine.

Can anyone tell me where I've messed up?

import tensorflow as tf
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import normalize
from sklearn.preprocessing import StandardScaler


def leaky_relu(x, alpha):
  return tf.nn.relu(x) - alpha * tf.nn.relu(-x)

raw = pd.read_csv('parkinsons_updrs.data')
print("CSV loaded successfully")

#Split into features and targets
features = raw.drop(['subject#','test_time','motor_UPDRS','total_UPDRS'], axis = 1).as_matrix()
targets = raw['total_UPDRS'].get_values()

scale = StandardScaler(with_mean=0, with_std=1)
scale.fit(features, targets)
scaled_feats = scale.transform(features)

print("pre-processing complete")

#Split data into test/train
data_train, data_test, targets_train, targets_test = train_test_split(scaled_feats,targets,test_size = 0.2,random_state=123)

print("Train/Test split complete")


#Need to reshape to match placeholder shape
targets_train = targets_train.reshape(targets_train.shape[0],-1)
targets_test = targets_test.reshape(targets_test.shape[0],-1)



#Network Structure
X = tf.placeholder(tf.float32, [None,features.shape[1]])
y = tf.placeholder(tf.float32, [None,1])
prob = tf.placeholder(tf.float32)

w1 = tf.Variable(tf.random_normal([features.shape[1],100]))
w2 = tf.Variable(tf.random_normal([100,1]))

b1 = tf.Variable(tf.random_normal([100]))
b2 = tf.Variable(tf.random_normal([1]))


layer1 = tf.nn.dropout(leaky_relu(tf.add(tf.matmul(X,w1),b1),0.2),prob)
out = tf.nn.dropout(tf.nn.tanh(tf.add(tf.matmul(layer1,w2),b2)),prob)

loss = tf.reduce_mean(tf.square(out - y))
train_step = tf.train.AdamOptimizer(0.001).minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

print("Training...")


minibatch = False
batchsize = 512


for i in range(1000):
    if minibatch:
        for x in range(0,targets_train.shape[0],batchsize):
            sess.run(train_step, feed_dict={X:data_train[x:x+batchsize],y:targets_train[x:x+batchsize],prob:0.5})
    else:
        sess.run(train_step, feed_dict={X:data_train,y:targets_train,prob:0.5})
    if i%25==0:
        acc = sess.run(loss, feed_dict={X: data_train, y: targets_train,prob:1})
        print ("Training accuracy: %.5f" % (acc))
        #print(sess.run(layer1,feed_dict={X: data_train, y: targets_train,prob:1})[0])



test_acc = sess.run(loss, feed_dict={X: data_test, y: targets_test,prob:1})
print ("Test accuracy: %.5f" % (test_acc))

#Check if still predicting 1
output = sess.run(out,feed_dict={X: data_test, y: targets_test,prob:1})
print(sum(output)==len(output))

Answer 1

Ok, I've managed to get a decent improvement. Currently producing an MSE of 20 for training and 30 for test. This is after converting all layers to relu and increasing their size.

The problem appeared the activation function on the output. I was under the impression that a sigmoid output layer was common practice.

I was also normalizing before the test/train split and leaking information into the test set! (though this likely wasn't the cause of my problem)

Here is my 'improved' code:

import pandas as pd
import numpy as np
from scipy import stats
from sklearn.cross_validation import train_test_split
from sklearn import preprocessing


raw = pd.read_csv('parkinsons_updrs.data')
print("CSV loaded successfully")

events = raw.shape[0]
#drop events where any feature has a zscore > 3 for that feature
raw = raw[(np.abs(stats.zscore(raw)) < 3).all(axis=1)]
print("Removed " + str(events-raw.shape[0]) + " outlier events")

features = raw.drop(['subject#','test_time','motor_UPDRS','total_UPDRS'], axis = 1)
targets = raw['total_UPDRS'].get_values()
print("pre-processing complete")

#Split data into test/train
data_train, data_test, targets_train, targets_test = train_test_split(features,targets,test_size = 0.33,random_state=123)
print("Train/Test split complete")


#Normalize data to mean 0 and std 1
#We normalize after the split to avoid leaking information to the training set
data_train = (data_train-data_train.mean())/data_train.std()
data_test = (data_test-data_test.mean())/data_test.std()

targets_train = targets_train.reshape(targets_train.shape[0],-1)
targets_test = targets_test.reshape(targets_test.shape[0],-1)



#Network Structure
X = tf.placeholder(tf.float32, [None,18])
y = tf.placeholder(tf.float32, [None,1])
prob = tf.placeholder_with_default(1.0, shape=())

w1 = tf.Variable(tf.random_normal([18,72]))
w2 = tf.Variable(tf.random_normal([72,36]))
w3 = tf.Variable(tf.random_normal([36,1]))

b1 = tf.Variable(tf.random_normal([72]))
b2 = tf.Variable(tf.random_normal([36]))
b3 = tf.Variable(tf.random_normal([1]))



layer1 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(X,w1),b1)),prob)
layer2 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer1,w2),b2)),prob)
out = tf.nn.relu(tf.add(tf.matmul(layer2,w3),b3))

loss = tf.losses.mean_squared_error(y, out)

train_step = tf.train.AdamOptimizer(0.005).minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

print("Training...")


minibatch = True
batchsize = 512


for i in range(10000):
    if minibatch:
        for x in range(0,targets_train.shape[0],batchsize):
            sess.run(train_step, feed_dict={X:data_train[x:x+batchsize],y:targets_train[x:x+batchsize],prob:0.5})
    else:
        sess.run(train_step, feed_dict={X:data_train,y:targets_train,prob:0.5})
    if i%25==0:
        acc = sess.run(loss, feed_dict={X: data_train, y: targets_train})
        print ("Training accuracy: %.5f" % (acc))



test_acc = sess.run(loss, feed_dict={X: data_test, y: targets_test})
print ("Test accuracy: %.5f" % (test_acc))

output = sess.run(out,feed_dict={X: data_test, y: targets_test})

error = abs(output-targets_test)

print(max(error))
print(sum(error)/len(error))
print(min(error))
 

I'd be very interested to know why this happened. Both tanh and sigmoid get 'stuck' in the 900s whereas relu sorts itself out and carries on.

Answer 2

The range of tanh is (-1, 1), but the target is at least 7.
So I think you should rescale the range of target to be [-1, 1].

And when I tested,

w1 = tf.Variable(tf.random_normal([features.shape[1],100], stddev=0.1))
w2 = tf.Variable(tf.random_normal([100,1], stddev=0.1))

works better than

w1 = tf.Variable(tf.random_normal([features.shape[1],100]))
w2 = tf.Variable(tf.random_normal([100,1]))

.