I have 2 datasets, call them dataset A and dataset B. Then I want to predict dataset A using dataset B as input using regression model.
dataset A format:
dataset A shape(15000,1)
dataset B format:
dataset B shape(15000,89)
I've tried to make a model using CNN, I have good training results. However, when I test the model using new data, the model cannot predict the data correctly.
Training data code:
Insole = pd.read_csv('1225_Rwalk10min1_list.txt', header=None, low_memory=False)
SIData = np.array(Insole)
df = pd.read_csv('1225_Rwalk10min.csv', low_memory=False)
columns = ['Fx']
selected_df = df[columns]
FCDatas = selected_df[:15000]
SmartInsole = np.array(SIData[:15000])
FCData = np.array(FCDatas)
scaler_x = MinMaxScaler(feature_range=(0, 1))
scaler_x.fit(SmartInsole)
xscale = scaler_x.transform(SmartInsole)
scaler_y = MinMaxScaler(feature_range=(0, 1))
scaler_y.fit(FCData)
yscale = scaler_y.transform(FCData)
sample_size = xscale.shape[0] # number of samples in train set
time_steps = xscale.shape[1] # number of features in train set
input_dimension = 1 # each feature is represented by 1 number
train_data_reshaped = xscale.reshape(sample_size,time_steps,input_dimension)
X_train, X_test, y_train, y_test = train_test_split(train_data_reshaped, yscale, test_size=0.20, random_state=2)
model = Sequential(name="model_conv1D")
n_timesteps = train_data_reshaped.shape[1] #13
n_features = train_data_reshaped.shape[2] #1
model.add(Input(shape=(n_timesteps,n_features)))
model.add(Conv1D(filters=64, kernel_size=7, activation='relu'))
model.add(Dropout(0.25))
model.add(Conv1D(filters=32, kernel_size=3, activation='relu'))
model.add(Conv1D(filters=16, kernel_size=2, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dense(n_features, activation='sigmoid')))
model.summary()
model.compile(loss='mse', optimizer=Adam(learning_rate=0.001), metrics=['mse'])
history = model.fit(X_train, y_train, batch_size=64, epochs=200,validation_split=0.1, verbose=2)
Test model use other data code:
Test_Insole = pd.read_csv('1225_Rwalk10min2_list.txt', header=None, low_memory=False)
TestSIData = np.asarray(Test_Insole)
Test_df = pd.read_csv('1225_Rwalk10min2.csv', low_memory=False)
Test_columns = ['Fx']
Test_selected_df = Test_df[Test_columns]
Test_FCDatas = Test_selected_df[:4200]
test_SmartInsole = np.array(TestSIData[:4200])
test_FCData = np.array(Test_FCDatas)
Test_scaler_x = MinMaxScaler(feature_range=(0, 1))
Test_scaler_x.fit(test_SmartInsole)
Test_xscale = Test_scaler_x.transform(test_SmartInsole)
Test_scaler_y = MinMaxScaler(feature_range=(0, 1))
Test_scaler_y.fit(test_FCData)
Test_yscale = Test_scaler_y.transform(test_FCData)
test_sample_size = Test_xscale.shape[0] # number of samples in train set
test_time_steps = Test_xscale.shape[1] # number of features in train set
test_input_dimension = 1 # each feature is represented by 1 number
test_train_data_reshaped = Test_xscale.reshape(test_sample_size,test_time_steps,test_input_dimension)
model.evaluate(test_train_data_reshaped, Test_yscale)
Test_xX_model = model.predict(test_train_data_reshaped)