src/qtable.py

   1 #!/usr/bin/env python3
   2
   3 import numpy as np
   4 import random
   5 import sys
   6
   7 from game import Game
   8
   9 learning_rate = 0.001
  10 discount_factor = 1.0
  11
  12 states_dim = 36864 # 2^10 * 6^2
  13 actions_dim = 539 # 10+1 * (6+1)^2
  14 num_episodes = 10000000000
  15
  16 def find_state_qid(shutable, diced):
  17     qid = 0
  18     for rod in shutable:
  19         qid += pow(2, rod-1)
  20     for i in range(len(diced)):
  21         qid += (diced[i]-1) * pow(6, i) * pow(2, 10)
  22     return qid
  23
  24 def find_option_qid(option):
  25     qid = 0
  26     for i in range(len(option)):
  27         qid += option[i] * pow(7, i) * pow(11, len(option)-1)
  28     return qid
  29
  30 def select_option(opts, qs):
  31     opt_qid_pairs = []
  32     opt_qsum = 0.0
  33     for opt in opts:
  34         opt_qid = find_option_qid(opt)
  35         opt_qid_pairs.append( [opt, opt_qid] )
  36         opt_qsum += qs[opt_qid]
  37     ran_pt = random.uniform(0.0, opt_qsum)
  38     decision_pt = 0.0
  39     for opt_qid_pair in opt_qid_pairs:
  40         decision_pt += qs[ opt_qid_pair[1] ]
  41         if ran_pt <= decision_pt:
  42             return (opt_qid_pair[0], opt_qid_pair[1])
  43     return (None, None)
  44
  45 Q = np.ones([states_dim, actions_dim])
  46
  47 running_score = [0.0, 0.0]
  48
  49 for i in range(num_episodes):
  50     g = Game()
  51     g.dice()
  52     state_qid = find_state_qid(g.get_shutable(), g.get_diced())
  53     while not g.is_over():
  54         opt, opt_qid = select_option( g.get_options(), Q[state_qid, :] )
  55         if opt:
  56             old_score = g.get_score()
  57             g.shut(opt)
  58             g.dice()
  59             reward = g.get_score() - old_score
  60             new_state_qid = find_state_qid(g.get_shutable(), g.get_diced())
  61             Q[state_qid, opt_qid] += \
  62                 learning_rate * (reward
  63                                  + discount_factor * np.max(Q[new_state_qid, :])
  64                                  - Q[state_qid, opt_qid])
  65             state_qid = new_state_qid
  66     Q[state_qid, :] = 0.0
  67     running_score[0] *= 0.99999999
  68     running_score[0] += g.get_score()
  69     running_score[1] *= 0.99999999
  70     running_score[1] += 1.0
  71     print( "%d: %f" % (i, running_score[0]/running_score[1]) )