1
$\begingroup$

I'm working on a Reinforcement learning project where the agent needs to navigate itself around the maze and get to the goal. (I used Q Learning as my algorithm) The agent found the optimal path in 50 episodes when using TensorFlow, and consistently obtained the optimum reward. The graph's something like this: TensorFlow Version

Since I wanted to use Keras for fast prototyping in the future, I tried coding the exact same algorithm with Keras. However, even though I'm using the same optimization functions(cost functions are slightly different), the AI written on Keras fluctuates a lot. Something like this: Keras Version

My backend is TensorFlow, and I tried switching to Theano, but no luck.

My TensorFlow code:

import numpy as np
import random
import tensorflow as tf
import matplotlib.pyplot as plt

tf.reset_default_graph()

def action(env, s, a):
    env[s] = "_"
    c = True
    if a == 1 and s >= 5:
        s1 = s-5
    elif a == 3 and s <= 19:
        s1 = s+5
    elif a == 2 and s%5 != 4:
        s1 = s+1
    elif a == 4 and s%5 != 0 and s > 0:
        s1 = s-1
    else:
        s1 = s
        r = -1
        d = False
    if env[s1] == "0":
        r = -5
        d = False
        c = False
    elif env[s1] == "G":
        r = 100
        d = True
        c = False
    else:
        r = -1
        d = False
    return s1, r, d, c

def reset_env():
#The maze
    default_env = ["_"] * 25
    default_env[0] = "A"
    default_env[1] = "0"
    default_env[7] = "0"
    default_env[8] = "0"
    default_env[11] = "0"
    default_env[24] = "G"
    return default_env

inputs1 = tf.placeholder(shape=[1,25],dtype=tf.float32)
W = tf.Variable(tf.random_uniform([25,4],0,0.01))
Qout = tf.matmul(inputs1,W)
predict = tf.argmax(Qout,1)

nextQ = tf.placeholder(shape=[1,4],dtype=tf.float32)
loss = tf.reduce_sum(tf.square(nextQ - Qout))
trainer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
updateModel = trainer.minimize(loss)

init = tf.global_variables_initializer()

y = 0.5
e = 0
num_episodes = 500

rList = []
with tf.Session() as sess:
    sess.run(init)
    success = 0
    for i in range(num_episodes):
        rAll = 0
        s = 0
        d = False
        env = reset_env()
        j = 0
        c = True
        print "------------------START-------------------"
        while j < 100:
            j += 1
            a,allQ = sess.run([predict,Qout],feed_dict=    {inputs1:np.identity(25)[s:s+1]})
            if np.random.rand(1) < e:
                a[0] = random.randint(0,3)
            s1,r,d,c = action(env, s, int(a[0]+1))
            #print s1, r, d, c
            env[s1] = "A"
            Q1 = sess.run(Qout,feed_dict={inputs1:np.identity(25)[s1:s1+1]})
            maxQ1 = np.max(Q1)
            targetQ = allQ
            targetQ[0,a[0]] = r + y*maxQ1
            _,W1 = sess.run([updateModel,W],feed_dict={inputs1:np.identity(25)[s:s+1],nextQ:targetQ})
            rAll += r
            s = s1
            if c == False:
                break
            if d == True:
                e = 1./((i/50) + 10)
                success += 1
                break
        rList.append(rAll)
plt.plot(rList)
plt.show()

My Keras code:

import numpy as np
import random
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
from keras import optimizers
import numpy

def action(env, s, a):
    env[s] = "_"
    c = True
    if a == 1 and s >= 5:
        s1 = s-5
    elif a == 3 and s <= 19:
        s1 = s+5
    elif a == 2 and s%5 != 4:
        s1 = s+1
    elif a == 4 and s%5 != 0 and s > 0:
        s1 = s-1
    else:
        s1 = s
        r = -1
    if env[s1] == "0":
        r = -10
        d = False
        c = False
    elif env[s1] == "G":
        r = 100
        d = True
        c = False
    else:
        r = -1
        d = False
    return s1, r, d, c

def reset_env():
    default_env = ["_"] * 25
    default_env[0] = "A"
    default_env[1] = "0"
    default_env[7] = "0"
    default_env[8] = "0"
    default_env[11] = "0"
    default_env[24] = "G"
    return default_env

sgd = optimizers.SGD(lr=0.1)
model = Sequential()
model.add(Dense(4, input_dim=25, kernel_initializer='uniform'))
model.compile(loss='mean_squared_error',
          optimizer=sgd,
          metrics=['accuracy'])
y = 0.5
e = 0
num_episodes = 100

rList = []
success = 0
for i in range(num_episodes):
    rAll = 0
    s = 0
    d = False
    env = reset_env()
    j = 0
    c = True
    print "---------------START-------------------"
    while j < 100:
        j += 1
        allQ = model.predict(np.identity(25)[s:s+1])[0]
        a = allQ.argmax()
        if np.random.rand(1) < e:
           a = random.randint(0,3)
        s1,r,d,c = action(env, s, int(a+1))
        env[s1] = "A"
        Q1 = model.predict(np.identity(25)[s1:s1+1])[0]
        maxQ1 = np.max(Q1)
        targetQ = allQ
        targetQ[a] = r + y*maxQ1
        model.fit(np.identity(25)[s:s+1], np.array([targetQ]), epochs=1, verbose=0)
        rAll += r
        s = s1
        if c == False:
            break
        if d == True:
            e = 1./((i/50) + 10)
            success += 1
            break
    rList.append(rAll)
plt.plot(rList)
plt.show()
$\endgroup$
8
  • $\begingroup$ Keras with TF as backend has known issues with reproducibility. What backend are you using? If you try with Theano, do you get reproducible results? $\endgroup$
    – Hobbes
    Aug 4, 2017 at 21:07
  • $\begingroup$ @Hobbes Sorry, I forgot to mention that. Yes, I'm currently using Tensorflow. I'll try using Theano now; I'll report back shortly. $\endgroup$
    – nedward
    Aug 4, 2017 at 21:09
  • $\begingroup$ @Hobbes Sadly, that didn't work out. The output graph is still very unstable. $\endgroup$
    – nedward
    Aug 4, 2017 at 21:14
  • $\begingroup$ In you Keras implementation you are not using you own SGD but the default one which uses a different learning rate. When I change optimizer='sgd' to optimizer=sgd I get a stable result, but always converging to -100. Not sure if this is correct. $\endgroup$ Aug 5, 2017 at 12:21
  • $\begingroup$ @AndréBergner Sadly, no. As you can see the graph when you run my TF code, it's supposed to converge to 93. Thanks though! $\endgroup$
    – nedward
    Aug 5, 2017 at 17:19

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.