CNN for time series: Input 0 of layer "conv2d_5" is incompatible with the layer: expected min_ndim=4, found ndim=2. Full shape received: (None, 2)

Question

I am trying to use CNN on multivariate time series instead the most common usage on images. The number of features are between 90 and 120, depending on which I need to consider and experiment. This is my code

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

X_train_s = X_train_s.reshape((X_train_s.shape[0], X_train_s.shape[1],1))
X_test_s = X_test_s.reshape((X_test_s.shape[0], X_test_s.shape[1],1))

batch_size = 1024
length = 120
n_features = X_train_s.shape[1]

generator = TimeseriesGenerator(X_train_s, pd.DataFrame.to_numpy(Y_train[['TARGET_KEEP_LONG', 
                                                                          'TARGET_KEEP_SHORT']]), 
                                                                 length=length, 
                                                                 batch_size=batch_size)

validation_generator = TimeseriesGenerator(X_test_s, pd.DataFrame.to_numpy(Y_test[['TARGET_KEEP_LONG', 'TARGET_KEEP_SHORT']]), length=length, batch_size=batch_size)


early_stop = EarlyStopping(monitor = 'val_accuracy', mode = 'max', verbose = 1, patience = 20)

CNN_model = Sequential()
   
model.add(
    Conv2D(
        filters=64,
        kernel_size=(1, 5),
        strides=1,
        activation="relu",
        padding="valid",
        input_shape=(length, n_features, 1),
        use_bias=True,
    )
)
model.add(MaxPooling2D(pool_size=(1, 2)))
model.add(
    Conv2D(
        filters=64,
        kernel_size=(1, 5),
        strides=1,
        activation="relu",
        padding="valid",
        use_bias=True,
    )
)
model.add(MaxPooling2D(pool_size=(1, 2)))


CNN_model.add(MaxPooling1D(pool_size=2))
CNN_model.add(Flatten())
CNN_model.add(Dense(units=119, activation="relu", ))
CNN_model.add(Dropout(0.65))
CNN_model.add(Dense(units=36, activation="relu", ))
CNN_model.add(Dropout(0.65))
CNN_model.add(Dense(units=2, activation="softmax", ))

CNN_model.summary()

CNN_model.compile(
    optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]
)

CNN_model.fit_generator(
    generator, steps_per_epoch=1, 
    validation_data=validation_generator,
    epochs=200,
)

In other words, I take the features as one dimension and a certain number of rows as the other dimension. But I get this error

ValueError: Input 0 of layer "conv2d_5" is incompatible with the layer: expected min_ndim=4, found ndim=2. Full shape received: (None, 2)

that is referred to the first CNN layer as stated here

Cell In [26], line 50
     25 CNN_model = Sequential()
     27 # CNN_model.add(
     28 #     Conv1D(
     29 #         filters=128,
   (...)
     47 #     )
     48 # )
---> 50 model.add(
     51     Conv2D(
     52         filters=64,
     53         kernel_size=(1, 5),
     54         strides=1,
     55         activation="relu",
     56         padding="valid",
     57         input_shape=(batch_size, length, n_features, 1),
     58         use_bias=True,
     59     )
     60 )
     61 model.add(MaxPooling2D(pool_size=(1, 2)))
     62 model.add(
     63     Conv2D(
     64         filters=64,

Answer 1

After days of attempts and looking to post that gave some indirect insights, I found the trouble and I can share the solution for

• using 2DCNN models with time series and not images
• avoiding memory troubles for preparing the dataset using TimeseriesGenerator

As expected the trouble was in preparing the dataset with the proper shape. The main bug in my code was this

X_train_s = X_train_s.reshape((X_train_s.shape[0], X_train_s.shape[1],1))
X_test_s = X_test_s.reshape((X_test_s.shape[0], X_test_s.shape[1],1))

that should be replaced with this (I also changed the names of the series, but just to keep the original one untouched)

X_train_s_CNN = X_train_s.reshape(*X_train_s.shape, 1)
X_test_s_CNN = X_test_s.reshape(*X_test_s.shape, 1)

Here is the full working code

from tensorflow.keras.layers import Conv1D, MaxPooling1D, Dense, Flatten, Conv2D, MaxPooling2D
from tensorflow.keras.layers import BatchNormalization

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

X_train_s_CNN = X_train_s.reshape(*X_train_s.shape, 1)
X_test_s_CNN = X_test_s.reshape(*X_test_s.shape, 1)


batch_size = 64
length = 300
n_features = X_train_s.shape[1]

generator = TimeseriesGenerator(X_train_s_CNN, pd.DataFrame.to_numpy(Y_train[['TARGET_KEEP_LONG', 
                                                                          'TARGET_KEEP_SHORT']]), 
                                                                    length=length, 
                                                                    batch_size=batch_size)

validation_generator = TimeseriesGenerator(X_test_s, pd.DataFrame.to_numpy(Y_test[['TARGET_KEEP_LONG', 
                                                                                   'TARGET_KEEP_SHORT']]), 
                                                                           length=length, 
                                                                           batch_size=batch_size)


early_stop = EarlyStopping(monitor = 'val_accuracy', mode = 'max', verbose = 1, patience = 10)

CNN_model = Sequential()


CNN_model.add(
    Conv2D(
        filters=64,
        kernel_size=(2,2),
        strides=1,
        activation="relu",
        padding="same",
        input_shape=(length, n_features, 1),
        use_bias=True,
    )
)
CNN_model.add(BatchNormalization())
CNN_model.add(MaxPooling2D(pool_size=(2, 2)))
#CNN_model.add(Dropout(0.2))

CNN_model.add(
    Conv2D(
        filters=128,
        kernel_size=(2,2),
        strides=1,
        activation="relu",
        padding="same"
    )
)
CNN_model.add(BatchNormalization())
CNN_model.add(MaxPooling2D(pool_size=(2, 2)))
#CNN_model.add(Dropout(0.3))

# CNN_model.add(
#    Conv2D(
#        filters=256,
#        kernel_size=(2,2),
#        strides=1,
#        activation="relu",
#        padding="same"
#    )
# )
# CNN_model.add(
#    Conv2D(
#        filters=256,
#        kernel_size=(2,2),
#        strides=1,
#        activation="relu",
#        padding="same"
#    )
# )
# CNN_model.add(BatchNormalization())
# CNN_model.add(MaxPooling2D(pool_size=(2, 2)))
# CNN_model.add(Dropout(0.3))


CNN_model.add(Flatten())
# CNN_model.add(Dense(units=4096, activation="relu", ))
# CNN_model.add(BatchNormalization())
# #CNN_model.add(Dropout(0.5))
# CNN_model.add(Dense(units=128, activation="relu", ))
# CNN_model.add(BatchNormalization())
# # CNN_model.add(Dropout(0.5))
CNN_model.add(Dense(units=2, activation="softmax", ))


CNN_model.compile(
    optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]
)



CNN_model.fit(
    generator, steps_per_epoch=1, 
    validation_data=validation_generator,
    epochs=200,
)

In the remarked parts the variants that I have tested. Regretfully, in this specific case, the results are very unstable in terms of accuracy and val_accuracy. The most disturbing is the accuracy of having really erratic behavior. Not clear why.