continued with qtable

diff --git a/src/qtable.py b/src/qtable.py
index 0ea19ecba7b0e88ae49b89e7cb46e83c3dd2ee25..fb1f3901645e0b64877d06320e2e8423b4bf04fb 100755 (executable)
--- a/src/qtable.py
+++ b/src/qtable.py
@@ -8,7 +8,7 @@ from game import Game
 
 states_dim = 147456 # 2^12 * 6^2
 actions_dim = 637 # 12+1 * (6+1)^2
 
 states_dim = 147456 # 2^12 * 6^2
 actions_dim = 637 # 12+1 * (6+1)^2

-num_episodes = 1000
+num_episodes = 10000000

 
 def find_state_qid(shutable, diced):
     qid = 0
 
 def find_state_qid(shutable, diced):
     qid = 0


@@ -42,12 +42,27 @@ def select_option(opts, qs):
 
 Q = np.zeros([states_dim, actions_dim])
 
 
 Q = np.zeros([states_dim, actions_dim])
 

+running_score = [0.0, 0.0]
+

 for i in range(num_episodes):
     g = Game()
 for i in range(num_episodes):
     g = Game()

+    g.dice()
+    state_qid = find_state_qid(g.get_shutable(), g.get_diced())

     while not g.is_over():
     while not g.is_over():

-        g.dice()
-        state_qid = find_state_qid(g.get_shutable(), g.get_diced())

         opt, opt_qid = select_option( g.get_options(), Q[state_qid, :] )
         if opt:
         opt, opt_qid = select_option( g.get_options(), Q[state_qid, :] )
         if opt:

+            old_score = g.get_score()

             g.shut(opt)
             g.shut(opt)

-    print( "%d: %d" % (i, g.get_score()) )
+            g.dice()
+            reward = g.get_score() - old_score
+            new_state_qid = find_state_qid(g.get_shutable(), g.get_diced())
+            lr = 0.1
+            gamma = 0.99
+            Q[state_qid, opt_qid] = Q[state_qid, opt_qid] + \
+                lr * (reward + gamma * np.max(Q[new_state_qid, :]) - Q[state_qid, opt_qid])
+            state_qid = new_state_qid
+    running_score[0] *= 0.999
+    running_score[0] += g.get_score()
+    running_score[1] *= 0.999
+    running_score[1] += 1.0
+    print( "%d: %f" % (i, running_score[0]/running_score[1]) )