# Train a deep learning model in chunks/sequentially to avoid memory error

How do I train/fit a model in chunks so as to escape the dreaded memory error?

def TFIDF(X_train, X_test, MAX_NB_WORDS=75000):
vectorizer_x = TfidfVectorizer(max_features=MAX_NB_WORDS)
X_train = vectorizer_x.fit_transform(X_train).toarray()
X_test = vectorizer_x.transform(X_test).toarray()
print("tf-idf with", str(np.array(X_train).shape[1]), "features")
return (X_train, X_test)

# In[3]:

def Build_Model_DNN_Text(shape, nClasses, dropout=0.5):
"""
buildModel_DNN_Tex(shape, nClasses,dropout)
Build Deep neural networks Model for text classification
Shape is input feature space
nClasses is number of classes
"""
model = Sequential()
node = 512  # number of nodes
nLayers = 4  # number of  hidden layer
for i in range(0, nLayers):
model.compile(loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model

# In[18]:

# In[19]:

df = df.dropna()
X_train, X_test, y_train, y_test = train_test_split(df['body'], df['user_id'],
test_size=0.3, random_state=42, shuffle=True)
X_train_tfidf, X_test_tfidf = TFIDF(X_train, X_test)
y_train_tfidf, y_test_tfidf = TFIDF(y_train,y_test)
output_nodes = len(list(set(df['user_id'])))
model_DNN = Build_Model_DNN_Text(X_train_tfidf.shape[1], output_nodes)
model_DNN.fit(X_train_tfidf, y_train_tfidf,
validation_data=(X_test_tfidf, y_test_tfidf),
epochs=10,
batch_size=256,
verbose=2)

predicted = model_DNN.predict(X_test_tfidf)
print(metrics.classification_report(y_test, predicted))


When I run the above, I keep getting the memory error: Unable to allocate 37.9 GiB for an array with shape (67912, 75000) and data type float64. I know I can send the data in chunks pd.read_csv( , chunksize= 5000), but what I don't know is how do I implement it here. My data has 98000 rows and 4 columns. Thank you.

• You mean tf.keras.utils.get_file("train.csv", TRAIN_DATA_URL)? Tried, could not understand how to use it. i tried tf.keras.utils.get_file("File.csv") but got error get_file() missing 1 required positional argument: 'origin'. Thoughts? Mar 20, 2020 at 20:39
• Add origin / TRAIN_DATA_URL Mar 20, 2020 at 20:43
• I get that. Tried tf.keras.utils.get_file("ExtractedData.csv", "D://Masters//AuthorReidentification") but unsuccessful. Mar 20, 2020 at 20:55