I tried two different models for a regression model.
The first one is a basic dense neural network. The performance is, for now, not great at all, but at least, it predicts always different values. x_train shape: (batch, features), y_train shape: (batch)
from keras import optimizers from keras.models import Sequential, Model from keras.layers import Input from keras.layers.recurrent import LSTM from keras.layers.normalization import BatchNormalization from keras.layers.core import Dense, Activation from keras.callbacks import EarlyStopping from sklearn.preprocessing import StandardScaler import numpy as np
x_train = x_train.todense() x_test = x_test.todense() y_train = y_train.to_numpy() y_test = y_test.to_numpy() scaler = StandardScaler() x_train = scaler.fit_transform(x_train) x_test = scaler.fit_transform(x_test) batch_size = int(x_train.shape / (self.nb_epochs * 1.0)) model = Sequential() model.add(Dense(first)) model.add(BatchNormalization()) model.add(Activation('tanh')) model.add(Dense(1)) adam = optimizers.adam(lr=0.01) model.compile(loss='mse', optimizer=adam) history = model.fit(x_train, y_train, batch_size=batch_size, epochs=self.nb_epochs, callbacks=[callback], validation_data=(x_test, y_test))
However, the LSTM model always predict the same value. x_train shape: (batch, timesteps, features); y_train shape: (batch, timesteps), but I only use the most recent timestep for y.
lstm = 10 first_layer = 240 nb_epochs = 1000 callback = EarlyStopping(monitor='val_loss', patience=8) batch_size = int(x_train.shape / (nb_epochs * 1.0)) input_tensor = Input(shape=[x_train.shape, x_train.shape]) lstm_layer = LSTM(30)(input_tensor) dense_layer = Dense(first)(lstm_layer) normalization_without_activation = BatchNormalization()(dense_layer) normalization = Activation('relu')(normalization_without_activation) outputs = Dense(1) model = Model(inputs=input_tensor, outputs=outputs) adam = optimizers.adam(lr=0.01) model.compile(loss='mse', optimizer=adam) history = model.fit(x_train, y_train[:, -1], batch_size=batch_size, epochs=nb_epochs, callbacks=[callback], validation_data=(x_test, y_test[:, -1]))
Any ideas of what I could be doing wrong?