0
$\begingroup$

Cross Entropy Method is considered as one of the simplest optimization algorithm which can be used for training an agent. I tried to train an agent to solve gym's cartpole environment and I have used this tutorial as a guide. I chose tensorflow instead of pytorch because of keras. Even after training for 200 episodes the agent is learning nothing and hovers around 14 and 16. What could be the likely reason that the agent is failing to learn?

This is the full code

import numpy as np
import gym    
import tensorflow as tf 

np.random.seed(1)
tf.random.set_seed(1)    

def generate_batch(env,batch_size,max_steps):
    batch_actions,batch_states, batch_rewards = [],[],[]
    for i in range(batch_size):
        states = []
        actions = []
        total_reward = 0
        state = env.reset()
        for j in range(max_steps):
            act_prob = model(state.reshape(1,4)).numpy()[0]
            action = np.random.choice(len(act_prob),p=act_prob)
            new_state, reward, done, _ = env.step(action)
            states.append(state)
            actions.append(action)
            state = new_state
            total_reward += reward
            if done:
                batch_actions.append(actions)
                batch_states.append(states)
                batch_rewards.append(total_reward)
                break
    return batch_states, batch_actions, batch_rewards    

def batch_filter(batch_states,batch_actions,batch_rewards,percentile):
    reward_threshold = np.percentile(batch_rewards, percentile)
    elite_states = []
    elite_actions = []
    for i in range(len(batch_rewards)):
        for j in range(len(batch_states[i])):
            elite_states.append(batch_states[i][j])
            elite_actions.append(batch_actions[i][j])
    return np.array(elite_states), np.array(elite_actions)    

env = gym.make('CartPole-v0')
env.seed(0)
max_steps =1000
batch_size = 100
episodes = 500
percentile = 80
completion_score = 200

n_states = env.observation_space.shape[0]
n_actions = env.action_space.n

# Neural network
visible = tf.keras.layers.Input(shape=(n_states,)) 
hidden = tf.keras.layers.Dense(n_actions,activation="softmax")(visible)
model = tf.keras.models.Model(inputs=visible, outputs=hidden)
model.compile(optimizer='adam',loss='categorical_crossentropy')

for i in range(episodes):
    batch_states,batch_actions,batch_rewards = generate_batch(env, batch_size, max_steps=max_steps)

    elite_states, elite_actions = batch_filter(batch_states,batch_actions,batch_rewards,percentile)
    elite_actions = tf.keras.utils.to_categorical(elite_actions)
    hist = model.fit(elite_states,elite_actions,batch_size=batch_size,verbose=0)
    hist.history
    mean_reward, threshold = np.mean(batch_rewards), np.percentile(batch_rewards, percentile)
    print("%d: loss=%.3f, reward_mean=%.1f, reward_threshold=%.1f" % (
                i, hist.history['loss'][0], mean_reward, threshold))
    if np.mean(batch_rewards)> completion_score:
        print("Environment has been successfullly completed!")
$\endgroup$
  • 1
    $\begingroup$ Apologies I misread your code initially and posted an incorrect answer. I need a closer look, I think it will be related to how you are preparing the NN data in the last few lines $\endgroup$ – Neil Slater Nov 6 at 11:25

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.