src/qtable.py

   1 #!/usr/bin/env python3
   2
   3 import numpy as np
   4 import random
   5 import sys
   6
   7 from game import Game
   8
   9 learning_rate = 0.1
  10 discount_factor = 1.0
  11
  12 states_dim = 36864 # 2^10 * 6^2
  13 actions_dim = 539 # 10+1 * (6+1)^2
  14 num_episodes = 10000000000
  15
  16 def find_state_qid(shutable, diced):
  17     qid = 0
  18     for rod in shutable:
  19         qid += pow(2, rod-1)
  20     for i in range(len(diced)):
  21         qid += (diced[i]-1) * pow(6, i) * pow(2, 10)
  22     return qid
  23
  24 def find_option_qid(option):
  25     qid = 0
  26     for i in range(len(option)):
  27         qid += option[i] * pow(7, i) * pow(11, len(option)-1)
  28     return qid
  29
  30 def select_option(opts, qs):
  31     opt_qid_pairs = []
  32     opt_qsum = 0.0
  33     for opt in opts:
  34         opt_qid = find_option_qid(opt)
  35         opt_qid_pairs.append( [opt, opt_qid] )
  36         opt_qsum += qs[opt_qid]
  37     #random.shuffle(opt_qid_pairs)
  38     ran_pt = random.uniform(0.0, opt_qsum)
  39     decision_pt = 0.0
  40     for opt_qid_pair in opt_qid_pairs:
  41         decision_pt += qs[ opt_qid_pair[1] ]
  42         if ran_pt <= decision_pt:
  43             return (opt_qid_pair[0], opt_qid_pair[1])
  44     return (None, None)
  45
  46 Q = np.ones([states_dim, actions_dim])
  47
  48 running_score = [0.0, 0.0]
  49
  50 for i in range(num_episodes):
  51     g = Game()
  52     g.dice()
  53     state_qid = find_state_qid(g.get_shutable(), g.get_diced())
  54     while not g.is_over():
  55         opt, opt_qid = select_option( g.get_options(), Q[state_qid, :] )
  56         if opt:
  57             old_score = g.get_score()
  58             g.shut(opt)
  59             g.dice()
  60             reward = (g.get_score() - old_score) / 11.0
  61             new_state_qid = find_state_qid(g.get_shutable(), g.get_diced())
  62             Q[state_qid, opt_qid] += \
  63                 learning_rate * (reward
  64                                  + discount_factor * np.max(Q[new_state_qid, :])
  65                                  - Q[state_qid, opt_qid])
  66             state_qid = new_state_qid
  67     Q[state_qid, opt_qid] = 0
  68     running_score[0] *= 0.99999999
  69     running_score[0] += g.get_score()
  70     running_score[1] *= 0.99999999
  71     running_score[1] += 1.0
  72     print( "%d: %f" % (i, running_score[0]/running_score[1]) )