Cross Entropy Method is considered as one of the simplest optimization algorithm which can be used for training an agent. I tried to train an agent to solve gym's cartpole environment and I have used this tutorial as a guide. I chose tensorflow instead of pytorch because of keras. Even after training for 200 episodes the agent is learning nothing and hovers around 14 and 16. What could be the likely reason that the agent is failing to learn?
This is the full code
import numpy as np import gym import tensorflow as tf np.random.seed(1) tf.random.set_seed(1) def generate_batch(env,batch_size,max_steps): batch_actions,batch_states, batch_rewards = ,, for i in range(batch_size): states =  actions =  total_reward = 0 state = env.reset() for j in range(max_steps): act_prob = model(state.reshape(1,4)).numpy() action = np.random.choice(len(act_prob),p=act_prob) new_state, reward, done, _ = env.step(action) states.append(state) actions.append(action) state = new_state total_reward += reward if done: batch_actions.append(actions) batch_states.append(states) batch_rewards.append(total_reward) break return batch_states, batch_actions, batch_rewards def batch_filter(batch_states,batch_actions,batch_rewards,percentile): reward_threshold = np.percentile(batch_rewards, percentile) elite_states =  elite_actions =  for i in range(len(batch_rewards)): for j in range(len(batch_states[i])): elite_states.append(batch_states[i][j]) elite_actions.append(batch_actions[i][j]) return np.array(elite_states), np.array(elite_actions) env = gym.make('CartPole-v0') env.seed(0) max_steps =1000 batch_size = 100 episodes = 500 percentile = 80 completion_score = 200 n_states = env.observation_space.shape n_actions = env.action_space.n # Neural network visible = tf.keras.layers.Input(shape=(n_states,)) hidden = tf.keras.layers.Dense(n_actions,activation="softmax")(visible) model = tf.keras.models.Model(inputs=visible, outputs=hidden) model.compile(optimizer='adam',loss='categorical_crossentropy') for i in range(episodes): batch_states,batch_actions,batch_rewards = generate_batch(env, batch_size, max_steps=max_steps) elite_states, elite_actions = batch_filter(batch_states,batch_actions,batch_rewards,percentile) elite_actions = tf.keras.utils.to_categorical(elite_actions) hist = model.fit(elite_states,elite_actions,batch_size=batch_size,verbose=0) hist.history mean_reward, threshold = np.mean(batch_rewards), np.percentile(batch_rewards, percentile) print("%d: loss=%.3f, reward_mean=%.1f, reward_threshold=%.1f" % ( i, hist.history['loss'], mean_reward, threshold)) if np.mean(batch_rewards)> completion_score: print("Environment has been successfullly completed!")