# Gym Cartpole not solving with Cross Entropy Method?

Cross Entropy Method is considered as one of the simplest optimization algorithm which can be used for training an agent. I tried to train an agent to solve gym's cartpole environment and I have used this tutorial as a guide. I chose tensorflow instead of pytorch because of keras. Even after training for 200 episodes the agent is learning nothing and hovers around 14 and 16. What could be the likely reason that the agent is failing to learn?

This is the full code

import numpy as np
import gym
import tensorflow as tf

np.random.seed(1)
tf.random.set_seed(1)

def generate_batch(env,batch_size,max_steps):
batch_actions,batch_states, batch_rewards = [],[],[]
for i in range(batch_size):
states = []
actions = []
total_reward = 0
state = env.reset()
for j in range(max_steps):
act_prob = model(state.reshape(1,4)).numpy()[0]
action = np.random.choice(len(act_prob),p=act_prob)
new_state, reward, done, _ = env.step(action)
states.append(state)
actions.append(action)
state = new_state
total_reward += reward
if done:
batch_actions.append(actions)
batch_states.append(states)
batch_rewards.append(total_reward)
break
return batch_states, batch_actions, batch_rewards

def batch_filter(batch_states,batch_actions,batch_rewards,percentile):
reward_threshold = np.percentile(batch_rewards, percentile)
elite_states = []
elite_actions = []
for i in range(len(batch_rewards)):
for j in range(len(batch_states[i])):
elite_states.append(batch_states[i][j])
elite_actions.append(batch_actions[i][j])
return np.array(elite_states), np.array(elite_actions)

env = gym.make('CartPole-v0')
env.seed(0)
max_steps =1000
batch_size = 100
episodes = 500
percentile = 80
completion_score = 200

n_states = env.observation_space.shape[0]
n_actions = env.action_space.n

# Neural network
visible = tf.keras.layers.Input(shape=(n_states,))
hidden = tf.keras.layers.Dense(n_actions,activation="softmax")(visible)
model = tf.keras.models.Model(inputs=visible, outputs=hidden)

for i in range(episodes):
batch_states,batch_actions,batch_rewards = generate_batch(env, batch_size, max_steps=max_steps)

elite_states, elite_actions = batch_filter(batch_states,batch_actions,batch_rewards,percentile)
elite_actions = tf.keras.utils.to_categorical(elite_actions)
hist = model.fit(elite_states,elite_actions,batch_size=batch_size,verbose=0)
hist.history
mean_reward, threshold = np.mean(batch_rewards), np.percentile(batch_rewards, percentile)
print("%d: loss=%.3f, reward_mean=%.1f, reward_threshold=%.1f" % (
i, hist.history['loss'][0], mean_reward, threshold))
if np.mean(batch_rewards)> completion_score:
print("Environment has been successfullly completed!")

• Apologies I misread your code initially and posted an incorrect answer. I need a closer look, I think it will be related to how you are preparing the NN data in the last few lines – Neil Slater Nov 6 '19 at 11:25