0
$\begingroup$

I don't know if this is the proper place to ask code-based questions on but I've been struggling with this issue for a while. Basically I am training a Deep Q Model using Keras and Google Colab (for the GPU) and am running into a problem where when I train the model, even for days straight, the model ends up taking a single action. The environment for this model is the OpenAi-gym "Pong-v0" environment and the model keeps taking action "0" am I not training the right way? Code is below:

#place to hold experiences in the form (s,r,a,s',d?)
replay_buffer = []

batch_size = 64
num_episodes = 200
curr_episode = 0
C = 10000   #number of timesteps after which to clone the model
R = 4   #number of timesteps after which to experience replay

EPSILON = 0.9
EPSILON_DECAY = 0.99
GAMMA = 0.8

#set according to command line params
"""if nargs >= 4 and sys.argv[3] >= 0:
  EPSILON = sys.argv[3]
if nargs >= 5 and sys.argv[4] >= 0:
  EPSILON_DECAY = sys.argv[4]
if nargs >= 6 and sys.argv[5] >= 0:
  GAMMA = sys.argv[5]
"""

done = False
t = 0
#path to save model to
path = '/content/gdrive/My Drive/PlayPong'
#load model from path
print(os.path.exists(path+'.h5') and os.path.exists(path+'.json'))
if os.path.exists(path+'.h5') and os.path.exists(path+'.json'):
  model = load_model(path)
  EPSILON, t = readTxt(path+'.txt')
  target_model = load_model(path+'_target')  
else:
  print("File not found, creating new model.")
  #build keras model, cnn with dense layers which finds optimal policy
  model = Sequential()
  model.add(Conv2D(32, 8, strides=(4,4), data_format="channels_last", input_shape=(84,84,2), activation="relu"))
  model.add(Conv2D(64, 4, strides=(2,2), activation="relu"))
  model.add(Conv2D(64,3, activation="relu"))
  model.add(Flatten(data_format="channels_last"))
  model.add(Dense(3136, activation="relu"))
  model.add(Dense(1000, activation="relu"))
  model.add(Dense(env.action_space.n, activation="relu"))

  model.compile(optimizer=Adam(), loss="mean_squared_error")
  print("Model Compiled.")


while curr_episode < num_episodes:
  print("E: " + str(curr_episode))
  s1 = rgb_to_grayscale(env.reset())
  s2, r, done, info = env.step(env.action_space.sample())
  s2 = rgb_to_grayscale(s2)

  state = tf.concat([s1,s2], axis=2)
  lives = info["ale.lives"]

  #observe environment until episode is done
  while not done:
    print("E: " + str(curr_episode))
    print("T: " + str(t))
    plt.imshow(env.render(mode='rgb_array'))
    ipythondisplay.clear_output(wait=True)
    ipythondisplay.display(plt.gcf())
    #utilize epsilon-greedy strategy for decision making
    if random.uniform(0,1) < EPSILON:
      predicted_action = env.action_space.sample()
    else:
      out = model.predict(tf.reshape(state, (-1,84,84,2)), steps=1)
      predicted_action = np.argmax(out[0])

    #breakout environment pauses after each life and requires the '1' action to be used in order to reset within an episode
    if(t != 0 and info["ale.lives"] < lives):
      predicted_action = 1
      lives = info["ale.lives"]
      print("LIFE LOST")

    print("ACTION: " + str(predicted_action))

    ns, reward, done, info = env.step(predicted_action)
    ns = rgb_to_grayscale(ns)

    print("R: " + str(reward))
    print("DONE: " + str(done))

    last_frame = tf.reshape(state[:,:,1], (84,84,1))
    new_state = tf.concat([last_frame, ns], axis=2)

    #store in replay buffer
    replay_buffer.append( (state, reward, predicted_action, new_state, done) )

    #replay experiences for training after each episode     
    if len(replay_buffer) >= batch_size and t % R == 0:
      for i in range(batch_size):
        print("EXPERIENCE #: " + str(i+1) + " / " + str(batch_size))
        curr_observation = replay_buffer[ np.random.random_integers(0, len(replay_buffer)-1) ]
        y = model.predict(tf.reshape(curr_observation[0], (-1,84,84,2)), steps=1)

        #update so that the q value for that action is updated, otherwise the other q values are the same as we haven't observed the other actions
        #add 0 index in call to y as "predict" returns [[predictions]]
        if curr_observation[4]:
          y[0][curr_observation[2]] = (curr_observation[1]) #R
        else:
          nstate = tf.reshape(curr_observation[3], (-1,84,84,2))
          y[0][curr_observation[2]] = (curr_observation[1] + GAMMA * np.amax(target_model.predict(nstate, steps=1)[0])) #R + gamma * max(Q(s',a'))

        #train on sampled experiences
        model.fit(tf.reshape(curr_observation[0], (-1, 84, 84, 2)), y, steps_per_epoch=1)
      #save on each iteration:
      save_model(model, path)
      writeTxt(path+'.txt', EPSILON, t)

    #clone model
    if t == 0 or t % C == 0:
      target_model = clone_model(model.model)
      target_model.set_weights(model.get_weights())
      save_model(target_model, path+'_target')
      del replay_buffer[:]


    #iterate state
    state = new_state

    #iterate timestep
    t += 1
    #decay epsilon
    EPSILON *= EPSILON_DECAY

  curr_episode += 1



#save model
ipythondisplay.clear_output(wait=True)
env.close()

The "ale.lives" is an artifact from when I was training using the Breakout environment, it just served to take action 1 - reset - whenever the player ran out of lives so that training would be sped up a bit.

Thanks in advance for any help!

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.