So I am working on a deep RL model in OpenAI Gym. I have everything else working except for my function that allows my agent experience replay.

here is the code:

import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense
from collections import deque
from keras.optimizers import Adam
import numpy as np
import gym
import tensorflow as tf
import math
import random

env = gym.make('MountainCar-v0')

gamma = .99 #discount factor for reward
alpha = .01 #learning rate
beta  = .01 #decay
batch_size = 32
memory = deque(maxlen=2000)
epsilon = 1.0

#building model
model = Sequential()
model.add(Dense(12, activation='sigmoid', input_dim=2))
model.add(Dense(12, activation='tanh'))
model.add(Dense(2, activation='linear'))
model.compile(loss='mse', optimizer=Adam(lr=alpha, decay=beta))

def preprocess_state(state):
    return np.reshape(state, [1, 2])

def experience(state, action, reward, next_state, done):
    # memory for replay
    memory.append((state, action, reward, next_state, done))

class agent():

def choose_action(self, state):
    rand_number = np.random.randint(2)
# choosing random action 50%, acting greedily with respect to value func/policy (naive e-greedy)
    return env.action_space.sample() if (np.random.random() <= epsilon) else np.argmax(model.predict(state))

def replay(self):
    x_batch, y_batch = [], []
    minibatch = random.sample(memory, batch_size)
    for state, action, reward, next_state, done in minibatch:
        y_target = model.predict(state)
        y_target[0][action] = reward if done else reward + gamma * np.max(model.predict(next_state)[0])

agent = agent()

def run():

for episodes in range(1000):
    state = preprocess_state(env.reset())
    tot_reward = 0
    done = False
    while not done:
        action = agent.choose_action(state)
        next_state, reward, done, t = env.step(action)
        next_state = preprocess_state(next_state)
        #record results


        tot_reward += reward

        #replay for agent
        if batch_size < len(memory):

        print("episode: {} | score: {} |".format(

        if done:
            print "You did it"


The issue is with this line:

  y_target[0][action] = reward if done else reward + gamma * np.max(model.predict(next_state)[0])

and the error: enter image description here

  • $\begingroup$ Possible duplicate of Deep Reinforcent Learning Model, trouble with replay function $\endgroup$ – Neil Slater Oct 16 '17 at 20:10
  • $\begingroup$ Your problem is simply that there are three possible actions in the problem and you have assumed there are two. $\endgroup$ – Neil Slater Oct 16 '17 at 20:53
  • $\begingroup$ Where in the code have I made that assumption? $\endgroup$ – hisairnessag3 Oct 16 '17 at 20:57
  • 1
    $\begingroup$ model.add(Dense(2, activation='linear')) is one place $\endgroup$ – Neil Slater Oct 16 '17 at 20:59

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.