I don't know if this is the proper place to ask code-based questions on but I've been struggling with this issue for a while. Basically I am training a Deep Q Model using Keras and Google Colab (for the GPU) and am running into a problem where when I train the model, even for days straight, the model ends up taking a single action. The environment for this model is the OpenAi-gym "Pong-v0" environment and the model keeps taking action "0" am I not training the right way? Code is below:

#place to hold experiences in the form (s,r,a,s',d?)
replay_buffer = []

batch_size = 64
num_episodes = 200
curr_episode = 0
C = 10000   #number of timesteps after which to clone the model
R = 4   #number of timesteps after which to experience replay

GAMMA = 0.8

#set according to command line params
"""if nargs >= 4 and sys.argv[3] >= 0:
  EPSILON = sys.argv[3]
if nargs >= 5 and sys.argv[4] >= 0:
  EPSILON_DECAY = sys.argv[4]
if nargs >= 6 and sys.argv[5] >= 0:
  GAMMA = sys.argv[5]

done = False
t = 0
#path to save model to
path = '/content/gdrive/My Drive/PlayPong'
#load model from path
print(os.path.exists(path+'.h5') and os.path.exists(path+'.json'))
if os.path.exists(path+'.h5') and os.path.exists(path+'.json'):
  model = load_model(path)
  EPSILON, t = readTxt(path+'.txt')
  target_model = load_model(path+'_target')  
  print("File not found, creating new model.")
  #build keras model, cnn with dense layers which finds optimal policy
  model = Sequential()
  model.add(Conv2D(32, 8, strides=(4,4), data_format="channels_last", input_shape=(84,84,2), activation="relu"))
  model.add(Conv2D(64, 4, strides=(2,2), activation="relu"))
  model.add(Conv2D(64,3, activation="relu"))
  model.add(Dense(3136, activation="relu"))
  model.add(Dense(1000, activation="relu"))
  model.add(Dense(env.action_space.n, activation="relu"))

  model.compile(optimizer=Adam(), loss="mean_squared_error")
  print("Model Compiled.")

while curr_episode < num_episodes:
  print("E: " + str(curr_episode))
  s1 = rgb_to_grayscale(env.reset())
  s2, r, done, info = env.step(env.action_space.sample())
  s2 = rgb_to_grayscale(s2)

  state = tf.concat([s1,s2], axis=2)
  lives = info["ale.lives"]

  #observe environment until episode is done
  while not done:
    print("E: " + str(curr_episode))
    print("T: " + str(t))
    #utilize epsilon-greedy strategy for decision making
    if random.uniform(0,1) < EPSILON:
      predicted_action = env.action_space.sample()
      out = model.predict(tf.reshape(state, (-1,84,84,2)), steps=1)
      predicted_action = np.argmax(out[0])

    #breakout environment pauses after each life and requires the '1' action to be used in order to reset within an episode
    if(t != 0 and info["ale.lives"] < lives):
      predicted_action = 1
      lives = info["ale.lives"]
      print("LIFE LOST")

    print("ACTION: " + str(predicted_action))

    ns, reward, done, info = env.step(predicted_action)
    ns = rgb_to_grayscale(ns)

    print("R: " + str(reward))
    print("DONE: " + str(done))

    last_frame = tf.reshape(state[:,:,1], (84,84,1))
    new_state = tf.concat([last_frame, ns], axis=2)

    #store in replay buffer
    replay_buffer.append( (state, reward, predicted_action, new_state, done) )

    #replay experiences for training after each episode     
    if len(replay_buffer) >= batch_size and t % R == 0:
      for i in range(batch_size):
        print("EXPERIENCE #: " + str(i+1) + " / " + str(batch_size))
        curr_observation = replay_buffer[ np.random.random_integers(0, len(replay_buffer)-1) ]
        y = model.predict(tf.reshape(curr_observation[0], (-1,84,84,2)), steps=1)

        #update so that the q value for that action is updated, otherwise the other q values are the same as we haven't observed the other actions
        #add 0 index in call to y as "predict" returns [[predictions]]
        if curr_observation[4]:
          y[0][curr_observation[2]] = (curr_observation[1]) #R
          nstate = tf.reshape(curr_observation[3], (-1,84,84,2))
          y[0][curr_observation[2]] = (curr_observation[1] + GAMMA * np.amax(target_model.predict(nstate, steps=1)[0])) #R + gamma * max(Q(s',a'))

        #train on sampled experiences
        model.fit(tf.reshape(curr_observation[0], (-1, 84, 84, 2)), y, steps_per_epoch=1)
      #save on each iteration:
      save_model(model, path)
      writeTxt(path+'.txt', EPSILON, t)

    #clone model
    if t == 0 or t % C == 0:
      target_model = clone_model(model.model)
      save_model(target_model, path+'_target')
      del replay_buffer[:]

    #iterate state
    state = new_state

    #iterate timestep
    t += 1
    #decay epsilon

  curr_episode += 1

#save model

The "ale.lives" is an artifact from when I was training using the Breakout environment, it just served to take action 1 - reset - whenever the player ran out of lives so that training would be sped up a bit.

Thanks in advance for any help!


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.