From e2c9b6f95875278586c544b43b6cce3db2fd81bd Mon Sep 17 00:00:00 2001 From: JohnJim0816 Date: Fri, 12 Mar 2021 16:54:22 +0800 Subject: [PATCH] update Qlearning --- codes/QLearning/agent.py | 9 +- codes/QLearning/main.py | 110 +++++++++--------- codes/QLearning/plot.py | 35 ------ .../result/20210311-192256/rewards_eval.npy | Bin 288 -> 0 bytes .../result/20210311-192256/rewards_train.npy | Bin 4128 -> 0 bytes codes/QLearning/result/rewards.png | Bin 18857 -> 0 bytes .../20210312-165244/ma_rewards_train.npy | Bin 0 -> 1728 bytes .../20210312-165244/rewards_curve_train.png | Bin 0 -> 34560 bytes .../results/20210312-165244/rewards_train.npy | Bin 0 -> 1728 bytes .../saved_model/20210311-192256/Q_table.pkl | Bin 5039 -> 0 bytes .../20210312-165244/Qleaning_model.pkl | Bin 0 -> 5039 bytes codes/QLearning/utils.py | 22 ---- 12 files changed, 59 insertions(+), 117 deletions(-) delete mode 100644 codes/QLearning/plot.py delete mode 100644 codes/QLearning/result/20210311-192256/rewards_eval.npy delete mode 100644 codes/QLearning/result/20210311-192256/rewards_train.npy delete mode 100644 codes/QLearning/result/rewards.png create mode 100644 codes/QLearning/results/20210312-165244/ma_rewards_train.npy create mode 100644 codes/QLearning/results/20210312-165244/rewards_curve_train.png create mode 100644 codes/QLearning/results/20210312-165244/rewards_train.npy delete mode 100644 codes/QLearning/saved_model/20210311-192256/Q_table.pkl create mode 100644 codes/QLearning/saved_model/20210312-165244/Qleaning_model.pkl delete mode 100644 codes/QLearning/utils.py diff --git a/codes/QLearning/agent.py b/codes/QLearning/agent.py index 3e9fb2d..f4a793a 100644 --- a/codes/QLearning/agent.py +++ b/codes/QLearning/agent.py @@ -5,11 +5,10 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-03-11 19:16:27 +LastEditTime: 2021-03-12 16:48:25 Discription: Environment: ''' -from functools import update_wrapper import numpy as np import math import torch @@ -53,11 +52,11 @@ class QLearning(object): import dill torch.save( obj=self.Q_table, - f=path, + f=path+"Qleaning_model.pkl", pickle_module=dill ) - def load(self, path): '''从文件中读取数据到 Q表格 ''' - self.Q_table =torch.load(f='prod_dls.pkl',pickle_module=dill) \ No newline at end of file + import dill + self.Q_table =torch.load(f=path+'Qleaning_model.pkl',pickle_module=dill) \ No newline at end of file diff --git a/codes/QLearning/main.py b/codes/QLearning/main.py index aa1e3b2..6fefb8a 100644 --- a/codes/QLearning/main.py +++ b/codes/QLearning/main.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-03-11 19:22:50 +LastEditTime: 2021-03-12 16:52:26 Discription: Environment: ''' @@ -15,101 +15,101 @@ sys.path.append(os.getcwd()) # 添加当前终端路径 import argparse import gym import datetime -from QLearning.plot import plot -from QLearning.utils import save_results + from envs.gridworld_env import CliffWalkingWapper, FrozenLakeWapper from QLearning.agent import QLearning +from common.plot import plot_rewards +from common.utils import save_results -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") -SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' -RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/result/"+SEQUENCE+'/' +SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径 +if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): # 检测是否存在文件夹 + os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/") +if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹 + os.mkdir(SAVED_MODEL_PATH) +RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # 存储reward的路径 +if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): # 检测是否存在文件夹 + os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/") +if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹 + os.mkdir(RESULT_PATH) -def get_args(): - '''训练的模型参数 - ''' - parser = argparse.ArgumentParser() +class QlearningConfig: '''训练相关参数''' - parser.add_argument("--n_episodes", default=500, - type=int, help="训练的最大episode数目") - '''算法相关参数''' - parser.add_argument("--gamma", default=0.9, - type=float, help="reward的衰减率") - parser.add_argument("--epsilon_start", default=0.99, - type=float, help="e-greedy策略中初始epsilon") - parser.add_argument("--epsilon_end", default=0.01, - type=float, help="e-greedy策略中的结束epsilon") - parser.add_argument("--epsilon_decay", default=200, - type=float, help="e-greedy策略中epsilon的衰减率") - parser.add_argument("--lr", default=0.1, type=float, help="学习率") - config = parser.parse_args() - return config + def __init__(self): + self.n_episodes = 200 # 训练的episode数目 + self.gamma = 0.9 # reward的衰减率 + self.epsilon_start = 0.99 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率 + self.lr = 0.1 # 学习率 + def train(cfg,env,agent): # env = gym.make("FrozenLake-v0", is_slippery=False) # 0 left, 1 down, 2 right, 3 up # env = FrozenLakeWapper(env) - rewards = [] # 记录所有episode的reward, + rewards = [] # 记录所有episode的reward + ma_rewards = [] # 滑动平均的reward steps = [] # 记录所有episode的steps for i_episode in range(cfg.n_episodes): ep_reward = 0 # 记录每个episode的reward ep_steps = 0 # 记录每个episode走了多少step - obs = env.reset() # 重置环境, 重新开一局(即开始新的一个episode) + state = env.reset() # 重置环境, 重新开一局(即开始新的一个episode) while True: - action = agent.choose_action(obs) # 根据算法选择一个动作 - next_obs, reward, done, _ = env.step(action) # 与环境进行一个交互 - # 训练 Q-learning算法 - agent.update(obs, action, reward, next_obs, done) # 不需要下一步的action - obs = next_obs # 存储上一个观察值 + action = agent.choose_action(state) # 根据算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互 + agent.update(state, action, reward, next_state, done) # Q-learning算法更新 + state = next_state # 存储上一个观察值 ep_reward += reward ep_steps += 1 # 计算step数 if done: break steps.append(ep_steps) + rewards.append(ep_reward) # 计算滑动平均的reward - if rewards: - rewards.append(rewards[-1]*0.9+ep_reward*0.1) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: - rewards.append(ep_reward) + ma_rewards.append(ep_reward) print("Episode:{}/{}: reward:{:.1f}".format(i_episode+1, cfg.n_episodes,ep_reward)) - plot(rewards) - if not os.path.exists(SAVED_MODEL_PATH): - os.mkdir(SAVED_MODEL_PATH) - agent.save(SAVED_MODEL_PATH+'Q_table.pkl') # 训练结束,保存模型 - '''存储reward等相关结果''' - save_results(rewards,tag='train',result_path=RESULT_PATH) + return rewards,ma_rewards def eval(cfg,env,agent): # env = gym.make("FrozenLake-v0", is_slippery=False) # 0 left, 1 down, 2 right, 3 up # env = FrozenLakeWapper(env) - rewards = [] # 记录所有episode的reward, + rewards = [] # 记录所有episode的reward + ma_rewards = [] # 滑动平均的reward steps = [] # 记录所有episode的steps - for i_episode in range(20): + for i_episode in range(cfg.n_episodes): ep_reward = 0 # 记录每个episode的reward ep_steps = 0 # 记录每个episode走了多少step - obs = env.reset() # 重置环境, 重新开一局(即开始新的一个episode) + state = env.reset() # 重置环境, 重新开一局(即开始新的一个episode) while True: - action = agent.choose_action(obs) # 根据算法选择一个动作 - next_obs, reward, done, _ = env.step(action) # 与环境进行一个交互 - obs = next_obs # 存储上一个观察值 + action = agent.choose_action(state) # 根据算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 + state = next_state # 存储上一个观察值 ep_reward += reward ep_steps += 1 # 计算step数 if done: break steps.append(ep_steps) + rewards.append(ep_reward) # 计算滑动平均的reward - if rewards: - rewards.append(rewards[-1]*0.9+ep_reward*0.1) + if ma_rewards: + ma_rewards.append(rewards[-1]*0.9+ep_reward*0.1) else: - rewards.append(ep_reward) + ma_rewards.append(ep_reward) print("Episode:{}/{}: reward:{:.1f}".format(i_episode+1, cfg.n_episodes,ep_reward)) - plot(rewards) - '''存储reward等相关结果''' - save_results(rewards,tag='eval',result_path=RESULT_PATH) + return rewards,ma_rewards if __name__ == "__main__": - cfg = get_args() + cfg = QlearningConfig() env = gym.make("CliffWalking-v0") # 0 up, 1 right, 2 down, 3 left env = CliffWalkingWapper(env) n_actions = env.action_space.n agent = QLearning(n_actions,cfg) - train(cfg,env,agent) - eval(cfg,env,agent) + rewards,ma_rewards = train(cfg,env,agent) + agent.save(path=SAVED_MODEL_PATH) + # eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) + plot_rewards(rewards,ma_rewards,tag="train",algo = "On-Policy First-Visit MC Control",path=RESULT_PATH) + diff --git a/codes/QLearning/plot.py b/codes/QLearning/plot.py deleted file mode 100644 index e64ceba..0000000 --- a/codes/QLearning/plot.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-10-07 20:57:11 -LastEditor: John -LastEditTime: 2020-10-07 21:00:29 -Discription: -Environment: -''' -import matplotlib.pyplot as plt -import seaborn as sns -import numpy as np -import os - -def plot(item,ylabel='rewards'): - sns.set() - plt.figure() - plt.plot(np.arange(len(item)), item) - plt.title(ylabel+' of Q-learning') - plt.ylabel(ylabel) - plt.xlabel('episodes') - plt.savefig(os.path.dirname(__file__)+"/result/"+ylabel+".png") - plt.show() - -if __name__ == "__main__": - - output_path = os.path.dirname(__file__)+"/result/" - rewards=np.load(output_path+"rewards_train.npy", ) - MA_rewards=np.load(output_path+"MA_rewards_train.npy") - steps = np.load(output_path+"steps_train.npy") - plot(rewards) - plot(MA_rewards,ylabel='moving_average_rewards') - plot(steps,ylabel='steps') \ No newline at end of file diff --git a/codes/QLearning/result/20210311-192256/rewards_eval.npy b/codes/QLearning/result/20210311-192256/rewards_eval.npy deleted file mode 100644 index 9bee5e4d6873b1ef653f02a96d352439f7a3eca8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 288 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+i=qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= eXCxM+0{I$720EHL3bhL411<(I&^o|4NNE5DG%d&g diff --git a/codes/QLearning/result/20210311-192256/rewards_train.npy b/codes/QLearning/result/20210311-192256/rewards_train.npy deleted file mode 100644 index 9395542137eb749c333d367ff082b27920a41e4b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4128 zcmbVO`8(9__g0pWB$2de5b~NaGiIN$mVIqPrtCyb5we!DMJugHB1)x2wBb#QZmq^v z6j3SCA}#bTEs8$#{{9Ky^UHmm=j6CAcN4ChB$FVs*(fwDc2$g#fMjI8 znrB2I8Lf_riHix242+5i4f`){9~==ICXL66gQLTw{vtM=PBEsCl1cw>QE;=_yrj z#1ff;@N1_Nq3Cv^V$qTWc-?);uD6WGVbfVnws&K3g5ogmizo)a;>ubKPH#ZPgS=w1 z!f3R|Kg}o>M4^-O;_Ti@>tQtE@V6i?93%OA7z@qU;ijVYeNUye*rZEm9-b?P=d$wq zorhP$RC0y6ryvX!ZMu#OPHR>LBhtHf zsP1nd#$|H1`_u*EQmf9U_SFHX^I(&_cllvj=%E^!t-d(*m9(LCu`f2O%4tXJU5Thz z;lP;dN;tPsS8v|90yduhOG>Ro7;Yk!mcJCiLO3*|YPtyFn?FAP?BflG(&IVNAC{x% zMWW7S(sB^5!so8<^@QP@io=c$UQn9L@ov!e!nKrfX7JE*oXKjc+}+}Z*B9rloAuKR zt-Q2AhY~M*{F@#1RnZH&SDKww7kNVNv(P4Xod>vGjl)MiF2mtHlcpLtE`w4*mA7u3 z5PBnuZ9|dEu+7qJyMmS%W^5Cy*Ts3lZ$^NB?63zOEr}GkzxIGA>JV$@4iB(Gcf2BE zm*G#DJ;S;|h%J(sAkW7_*vqjUB>78GCUp8biRO-wJl}?Ne|L08bc`A0x?}%u@89|d z-SEJ%?RQ;`8?MGFNT$`hKy~c`u}ZlMe9E^HSy!A9-g-u*$;$-}3z|19&0dPH;)kvp zS#D7MscUy|z!eI!=9RQX2vO3<*yrgegh#G#p=9n-T;!E>7c$%tuI+dBUr#sm%~D_( zopMFKMv46i4OiGo#?%KU3!%Z?zcZ}X9f1>7_Zy`3!>{#@^dEDD{JUEpn%#w{EehH3 z&rBg)+){AykUQ>dw5;Iix+9riE7qRjh8s+k@>SDaL2eYKl^tcKEjXY4^rbTb%VfVEm}u2@?@T-OA}scuoC!YnP8BSm#cM zyh?OHUMl^ji@F1jS=j|9KD5WV1zDxV*X*IqNxrN4+z!dfH!j7Vx5bx{{dJSh*g{36 z2&{ZtSZ>Jf({!{&EyE?b4>quN@KPj9}nwK zmV=97srAXMqt_Zbjn^hCM=r)()^l-`WHB!PJ~-pj+{L)6*}l)iYB3t_b-r8m(i)Y2 zdop69tWon<&ENf`HFmr1BlXR+Mvy-H@vkl`C_kdbBsEy#x-z+ShlUj*13c$hC|Sd# zH>6<74@>AKM}*&Yv<6q%G092Q8e!(Acbw6-#`}_cdqVmxvCX(3^^}$s4t6UUI_6m+ z^SRDDLd{58Y)P-0}^8#5$(b~wcpnql{)6Y*&V7C7|rJZDd&1#}fBr@V18N2WmhrE1h1 zHf_Bz*AvVTtw3ejw40$t{`ThUdS)p3kXL$kn;E8ieJY=AC&20Eooggd_;@X(_O|&7 zAhRXGMDS67HRhSOj(7?1g|t#fxLA5{s94C&w_TCRR)Kn@=h zvgI0`|M1aVvpTRRi-(#gOU!%6xe$#iW>jeLAWrx^I!NWipF5IJyq|~l%^TVq3VGOa zRJ$wpUoNgs(=w_97k=5nT|GHm)P;VrP7LQ_+TtEkZ4eJDcia|QDf95~*M&xTZ!VN_ z+t=`p@X%RoP82-h0=}|kU8+2c@i-yh47uoUp&Lv;!NtTfbu*()T!amLPw%SZK$RKr zY}z*tFgE37%pwl@azFi+v1DUhe%FOoE(hZ}jYIN(+0a`0?xRjB2aQ3prQ&}%a95VS z`@>LLulK&G{5l)-h;Z3S{%rh8s+d>xh=rtU?6jTrEbJVW^dyzDko#Rs>}+MB?%aXw zF&+!CZ15jg2MZggUk7EJXbfPMdDeMs{g_AZe2|B%(z1 zDLz(UPnQ2tvd$8#28Vv^+hl<~TvA}0r#YBmve)^u&EcRb`<=PP46%l5{TrhMC@g(J zn&cwY?Pu`1cW3!fa;ezsKip#xhww(#NXnbDf1fG9@h_#6fS){^Q~C96M<5R}27R}J%(-}XRiY{4a3DKcn3L|z#`{|_ z#c{@LgkNZFsLqn={7*-&{%aPNcaO-E=CE+()(UyyYw0~~SzFxCl=9O%sOr+lK(>bv z32PXbt)*G<&vz-$`+?O{1Ek38x^D5=6irkkf4OK=)YzOLahn-fICAjeYYqdX?)iF? znoTiu=3Tp+&=h=^`O)X8rkF##3vK&Ohl|QXAI}4HoI9z@n_$yX;HwmN=speMMd$iB zX>HZWDip@m*JUIw44g<3a!Ypw^WR~Yp$ZL zrNUsGwtZ&^9g6t}?E0tD5r4L1;CCerLn+T)x6G$O!78WxdMgzlKdd|QeH#tSAOG6@ zBY=i0-F=c5W;8?^Yt4IAO@&rWWvYt>6`HN@|GoXt1VQocG4;s6~fq~+t-V5khrf*i1>Q^29%}iISpXmOg_eagCnCxa*Yb4dLro7!?ccfH@cTqbp%rb%O{N}Nv z_bJGb8On`Iqu?Af;-lD_f;o-%AKy`+AmvEfO=ms@-J(4&{bVSxncNaWY9&K?vQD;a zi4@uHrWx^M=pXuJXDlRxH06F)7L^P+5;N3Ujf~>8xtKmmV0xg3$=>G#77ZU7O}$6p zq($Uy6>Tyq*UcLE@`u1fN^WY|0D)b;@@21D2uOJ0d118#wrVFSjGiK3?EH9}UmgL2 zv)wNuwi7Tr?3ed@BY_9YA4R;1AP^~&^?M?iK+{`^BVz@D`8CyXK}!i#jVP^+b|4UC zaQ%RbHG!1Rw~BiC1iF=+tTk8!67D#6*)f)2z);va3<>!_&Kj| zA9M)JJTle2VlDx@lNR>|XG_n|Y@Z!Ii@@1{VVa2sfdaFI<%`s%yx4+Pks5)@L+`?? zRi(TH^ZU)G5xAb-I{tntf&C5g6Sc|&)&}!Z%9IFfT3E2*f+B(Sf@tmbDN^2RT*eip zSfuu2z)OL^o<-!=%kl)?DnI=|k|HNMaf)0xf#yp(>xDrCuk7kmmM;Nc*7aLt!Mz9Bnu zp9g{APTGK*Gl4AY7nffyA#mwY?p2yO0ne-jz6Xj39Hy`3z0W4#ZTIs{dkO*8)L-uU zF$A`ZoDZs7E$!dkb|Th~0K-_&$MPi5|D{#(O4@HWN~*(b2u!``MfNJ@LYdd{HDxac zN-f+o-34rvav7m#Z?JH_`Cjbo1C|&R7PN&tFh@~-fK8pO4OH|xhgaT~?vuQ-g>omX zq0Au1PqDIsMPJ#xPvI6=w@Om5^S&AStk|O9GyyJtyfCTWkdF@C4{K(4^I#E9N_@`b zVwuU&_VZK@{ydA_RcOOTlX~^HYLedEY>Qw+M266@8a@6AS8-o|ruQ0j`pN?hp>y1C8w z7@2mTiftMQyrOL{Y(3PyqoGDj>mtS|B+|RuM@`4wAJMZLxv2Bq$OzWHG*O@&kE?6daXYpq@PloZaaW#we0 zP$+AuXHTh6C@eQ9lqGH7t-${g`P9^jzeMe1&fBS4Ua@n$Xk$c?zi4N5)za>&>80%s zMmDylmKFzv4hkLGyWPaj&dOFySoqq%JwV9P##s0VwU1eNkyTb_wQMOAwu|J?l4z+Y zQwoK;k$UR5+V$YUW;fR|=HlYwY}w_@hu<8cafa7zs;7-TRO5Mm-DAH?p#Lv6?z9xU z9W^wYprB%E?&Rqley;`LN^q{K8jTSFwhB-rF4N#vF2W z9X?;r3rXze^6f~wmCfk4-+F9sN%AtR$MAZ9;}ZOxk@ejT{B`Fi3JZm@=?>*P3gyBk zmhBYEjf?*)evvNXGuxD2gNM5nHBOznt^Mk1%wS_Whv*S;aUHopp_-*^ z$DRcU>hK>ppyDxIV&M1NZ_noD=GM98zxUo^zA!g!r0)K>r!w;XF2xYt79&cAY)(lw zg|f$TabYq)&7xV%d(-|CjxA167TJTZPIPY*cQ}`nlq4%BSM}RZC##c<<-t%Fl{j~|IR#qgaxDGy0_^k|dk-Mi*})iE@o+aa3|Ug;?lA67em{?W|jaAHIR50&F}**3?f zu%v8C4JwwdCFCru3D0b{@meu3w z_Q-OJ<}5xjv8)Pv8jY6fGHD?neDo9FoZf~Xe+dg|T$0GVmUUtK^Aa}; zUoMGQ99;iHMafV26Z6Y2-^rXlZPt)#D&{f~Z`dlO&mz<->Ez_(b^pHV-S4+;;Nqf( zoxE!{Kf^R@%1BbxqcpKC`C(r{ta{vrjT>Lc+~3&y`$}##1qGi=moB9`ja~8HeDL(@ zjeC`zirH`2vLz1t=;Z7iiz8F`>GI@IvzW(1&csl2|96VJ6+@~Q=4WeSRCj9yL|%Mz z$F%X)HL_#v9UUSzoo9J@c}>53e5}B99WB}&tDP3RQCL5|uI_yOpu@ZO@6AdA1cMHl z((s6I%e?8+_wL>EaT{>RzO#1A3!Fx8SD-=dbD8^{!^4T6-`ws0`uUB_grwp7HMvvm z_xoxSo{$$ierttkSFu0m{!6#_mQ?=o?e~rQFGus6Qj1$$R@de5GEUIC(N*{ceTx0Y+Mhxf4Omf~9Qd!@|O9Ck7h@OOoS7%xlko{`|Q% z)3!UlqtI6eAy-S+%MQ+C*p^EeCsgv9ubDvn(u1#xk9g~k5 zVfHh0$OH2in8H}7`CvnuhKWhEedmV$@`T{OzI|$kxz4^ z<@%fCc-pr60!|jbTSYIC-(u;G4M)Vp8nTA-X1Dr^I}Tly4>}a{c6O_zYaNNYw5#<8 zqC|ah3PNl0t6d%ux2Q--8EX z4ucIvB_(nOZcKk3)7go^tnLt(>SxE7V$E{2RQrC7X6wRjvAUVbKmYu5s_V23fJgR> zFIUa0YmKRw-m$ukl^!WBEw!InNN=*a&#xZac|ABdcrQ0Mcba8u%YKbWS$6w@x=$k+ zS4xA->RyJuPzt z_69_CdrGNXJN$%4G^Fvqfk$41p#Q*m-6C5TWx!xC7b8Zsu-C@oqF1AFq0hGDy>gRH zw!T%tvqRa5bs|eLR<p~7#SI{H+WpcrHf0@N#_?A&z7F&*d}Uoinev~yM^g#{4?UTH)q!& z4Vs}co?qf_Aw0@t+B*;Q_VSe^vp&_Kxm8wH@=$Nyys2QjFt_Q3%MJ?5idc<=AY7Mc z85tQT(iw4@inQt^!!rrGnJPs^MX_ozv>*}l7vX1o&EDKu+kovqaNt07rmZoRqpTgl zMJ4DgW*3cP=b;Mc&n4#}&0OkV_1$+By;V_0DfOK=rA_%$LrwSbknxWP%gKFj$oI%! zm@fC|OGs}~k(QPYn;gv+9!D3vB=*R!3fopQyyiZjieA@tIM>hsS3SUTM%>5giYf@m(^ zmrMJWwY6j{TeE2ol5hU}#1odKOW8MW{4&+>>e`bQ_t|S_zU!}!d8iP4G^{30GtQ_a zV02jwKf9<_JZf{ZC52WZc%R>ngtw$PdSzxz>=BGqMc@@-rgNA|KxDi{6r%=2S=1)p^xbMLOcUFWHvq8<>l+E3D~d6ur{RHI9t4V z{d#M8NvL^Fby5bcyH*`9z$+{)?6xr7t#{buO`X_DcK{RNm`->~(M6mm1x z9%*`(m^UM9nad1V!Nx@{AXNpJ^Yg7+X{J@Xytf=tsPrs}`RS*hX0S?;OK-m`3<5CH zB8)@bG?2PfPoKV{k)RzpKRYQ$D+`g(>0G%s-L&d7V4y11v}$~II6n@lR6EZ#6QQ;> zaD!#`;9*KngyP<%9mbudi1-x8K`6js^546>?Ppiw|A?@Ix-MRwwd%KKs6EYos?1a%J}Ddv zeYqW7Bg#UD#h16~AB&f3UcQF1=f{%d@uJ^fHa0fZVJBBMJNGAe-Mgpk&#yi{nBHoJ0+aLb)kmj& zwraKTvg}#0|0~W zf;M5_8a#(muy2_(dWAPXQ1HLjoc!bWi|kG8z-)(X^Om!Ny}h54)0)@jYUu7_$|bS~ zQs;vz;-aGNj&Q4=qU4-#$c{`+9XMdXXlztnvP9kgv6II1wC19ebAC`yTKj<*1LuT{ zbA1dyeOzOHVd0de+XHX>>8<|DHT)0!;-koq6yuMN!ipa5=GbJA?U3Ni?2ANM6zwuO zq@86KbM>&!z1m}TJ-_YPvE$pZ*k@v^EJ_oWK0}Ud%CzsSaQD zMEzWrM#NhpsyQ)l6s`P7V|Dh9gAI`=(@Gy7j5^o999^MC#^B}xl#cA#xk!-;zA4Gc z<0yfnfmt#`4!@Zw>#4DoZPqOkuP4L zJh>?5JRXhedUdYuUTQxhQPajISrX`^!q7n{rE=v;eg5OOKpl7YyyB0?c0VcA|I(km z{#)P$>I(=xoDqrCM5EYIZWAC!86O!@K6~~9d*VYJ!XITh=~TQqMfh!o8q&;}1XGd6 z6gmzXm#JO4^c?wWu!O!-s?}b2-ILOxN@nw_KVOFGxR|C1t^zqxo6cVVm7U$)QBzY> zaxGOH$SPyq<>KDlG71G`HSwF0pKOtH0bq|sqrdB0U!a2KL7OebQM+AZH;C% zvAk5Ym=Aq@eE~@?o;`bZg0X7Nn!Pq#C=_WKJNXp_0Cp4LHZ7th*M*~Q;SHPASsIt? z*W+l0Mjr?xK0K`Cmld!JXdIRJo|V<+c22wN$UBaA7Q#iG_$9k{z2!}E`G5EyxL^M@ z!2G{?b6ac-IvkVH&Z6b-5xQ~uxy~_Y1p8WYvjc?m_@9(&3}}<^La#}B#1ILWKD3(F z>h7yZlEUWPvD>oIo{F;oNOO||Yg-R#)v#wqR)?=ajlSPU4TGD{iEx+itFh}^ zMZ*;I5`26y1@*b@myM0hkyeX~iax!)w|@0g+bOK>5 z`qxcisOiq2(&N)xzbK`s$hLf$T5i0X-0SjkhLL&pmOo$d7)@1wIm#MVRu$aB2aRz6 zkoGj(Dv_&o`?g6qtBs6|V9okrW2j?!sMuIS5N)SsA>}jYCtD4aDXEQuMpr`c`u97! zy7&MX_~~Kk72&b}aN{fE0AC}^s!*^5V()euKrV=VGH{dXqbn>Ia1MTAx;YgVS#qm`KeH>I?GDKjX7i`P{i* zUESQ6>G|^sbzgT=QKPEY#HyzmmvcppEM56MDYNK%*^#;vG6tVW3w8BIg=&E0`BL^& zz|ztZh$;38kC!7H&n^^dzXWI#R)bZZ?TP^N3*=*AXny{@ed)Lx7 zWB}4;QAypu`syM|5gsa=?e4^51Yj*inj z280qFd%3rSe|QJo=YJB> zr~KuA9Z&hs_*VKo%2BX7@*&4E3#O2D&;|(;whb*0qcTzsjhvS*AvY8A7qY}0hvGqZ zt>rv)w$*($VYR?Y3dKswVjE@9AseS2cl-8jmPXPJKR#$|2HK?I<;7d19zN3Qs)}}3 z$R_vQeGYTLi1ScpcdGlmbF=GMsh78R4GJH3i%s*qIHPU(m1HwYp8l=w&-(^}cv=2#hTHW$TS7DL1{lwzLhUDEqo~&j~Mz z7ZiBuHa9gw`qrvP=q}LGdg&hAw@=ByeLB*IO9III0;P(2tQr+9F5lP;z^KXp8$$k9 z^~yo}J~atuwQ65OT3k*}PG@g#4639!+zlg@gSZ=M&vaSZfPOXTPw{dOq?pYP1MwUP}KotbuqRTyDXoS9|>eFQY=)o7KqW4RVSG-eMdMmRe; ziOp+2`1IEsTlNNYmzhUZPR=nxgvwZ0Ry@0j)ok?*4-Z#Qx6mWSS3y&CwHmsvbEIH$ zbEV9PAxS4);ZA6#$ zbaaIO_SdxV7(rKPX7o0nNpxnuMFw^vfK?c5ws{IxiYH9x!{IX_+n{;38< zau?VX>_8zy#?;ibxVV_c1N?~tDiId+m1QeE)MG&lM4zgP3XRKUAZgSLXwVj$y{nuO za@6_+NHsqmCOW`^32yhPMIsG4VC1Eds6R;Cfg7acAj?kS!2WN5-knNU(E@2@`C65`{cKSaf4J+;)m`+QEFa| z?bz2FJiXb&c~$7)iG~umVHGwzSvB_H4g5aUhg*v1A~B#eI3-;(cuZ0IlQ5bZ>CjW* zAs=4>AHU^#hqvsxbJfH7i*%4>V(&SS*BULd<6Prf=nXC|E``sEH0@Lq zWep~tT<}h4GbP)ucLy7r?>gZOgpB|{v@r_ZzKF7E^XA^~Za8G?fhAG{nMyc0l0|!a zdr6)Ixk7rgK#nz7u7GV31jZ2r9u~j1svYmFG=U@Hg}4%4F=F5`uMXm(nVo}Zc?^Da$^i79 z_V)JFYmJ8s9~o)Q%#D>Ju#Yj(VSZeV2T9lS*Ay0-@)zfaA<^6iZ)k3G`SPCa+oc_wtnZ@l>xktfTXzRb zf_@T<)(AWZ6>^A;G8HSbid1P{I#qh4)vxbl7#`T>jt8RDOa!(8e#+JV*kW1xRa1zL zDs;u)xDFnkQ^0lWhM~`&!${1Y`}@i_7oaf~*s+QE&Q~1+$RLSbfy4vahZx=igCFaNHe3fU=lwsXh6BlTaGjf_a+-d;1L4jLk-dHYR?1tu#2lXgsO4!tFQWcd)v=Rf>;1> zW|B47bbdI&m_-fC#0M02(+@*CX^}sV?Mjmai7fCitHjGlODyzZVzBNL6ri@I&3bhM zzFqdTJv0HHE33Os45k6(_gtEkF4a?yJ;t%+HPfWSE zPcvpl&N@G)uSSlJP|ocO+Gs2OQ^j(`5+N-18@@L|1*4s56G?O|`A7SX0|IME-BJ4F z2*>7v1)@>DE0dFz*VFJOR z$&Tl;cJ%g!0r~@TJudDop`~lq-;u@D7inqH2Pq*-vAe0VyZ{7-h(*(Rnq7%tx(O&Y z8V@+$wOfxW3FnTh+^b#nm*ygfklLFzMemXi+9N0^5ACo*RZ&8Hdk^7XX{@?gWkp2~ z5)&=&<+p~`|64wOEEeHUkJF_-*6O~Tu4R;$Cq6Ph`kR3Wc(NPk`u?CR?9=rHrmyo$ z==h^bsMMTYNk~P= z%HRllHkm_-vaz-G4+){8(iAVvzur69S@ck$br%0+Wo6BJuB~ewdK;BGzd?6`t+9l^ zrT?PaCVJ}?xM2M#64gAk!{VIYG6n_y3eEUQE89{7AKgUqYsF2RF0)g z;m#G8mn)LOm&f$uqy5eh#jCUI&D}uOI!}GRtEY1LBA8kKM~|Y>){wk31J0jdFen_^ zkPwarSw%%nXXh-#&@)_4r-pOJd`O6((Cb5|REsK+i4sCTEejY{tP>-&WaYADCV&Dl zr_l(W-i6s=P3&9V{Dcn56oCT=DtTo#zCh^)R=Y~aGB+|%-@j z_-X1)KHEzq&-~=%^~@}M=2U-zcb#Sx$Zx;Nlxp}n(p$L9ZZ)!d-gYg<@4vytJBOi zq2VPJY!SlNwdEl|7Ds$NNKoMLYk>2X@7}#L1(=c+7kscWf%A(OFVbVtJW`CFRd-97 z*ThB=*1jdjNmf?2QhYdPr-6Zipl*f|1P4bMZ#-_#$w;xKd-SHZccIKA%o9Q=5O*yc zycHi{38Dc#`bYAD(P~SqNcWNLn22bzuaZB)hdPZ=*{051q;RZAzr`yQUNZf8k)a|S@j2X z5t7~4`14cT5tU}GdAWoNLEsmamzS&CBeDu)@EZ9ueF9J*&(aVVerT_@Smr3}>+6q> zj@nRflI7?`&dhCuFgaM4bdqcoJ_Tt6=Rxao948CACUJI$0lP_RP){Ed5Wxs z+hhRlHAp|j^aJ>H#;F*V3jh|spdjs@S7(`D)A)b6?;b#oiw~ zcFb;W%34q>`4qO5|L+^f+`kVLLlr2r>U<9mVkDp)au_(jP0C&ONR!pA_deVD|Gh{> z8Fk3KcJH>MR?>vK!rdk0XHSm_bFfh}FE5WCyB_atgydv}e3peWq6ROD{MxE6LiXBI z)}HMGL}PRlL$TU;s1>_k-p(L)M z>*>HVp)kBGucth8Eh!2oRe?oFg&DEm<0oC-$gGv%s51YovH$P(4Vk-G+J4=9 z5RG&vONW3)yb{DmNT!5@G(%#2M~8y!lsnTGM*swZL^a$NxE%HBMrfBZc+(}EhKF7;#%nG@C_in&mt>@ow6Fchd0x{#@ z!IS}%NE9p8mM%SMP){)_0o_h1)>jz+d^oD)5EJoX1Tb9G3?Y&`71XF2Nqo33Ipt2#gW4)r}hzR3V>f zv*>6QiDpO&epEGq852@tCI!YDQ7;)7-jzJp@} zM#Bah@F}LJr}+<#1z$bv@;Gr=;jaqY{G6+|REd_*k_xmLP=_>A-y`#}Z`~S?VqEM! z4lur{NtYxiAeHCPB+)iO%b0;RAZ8$l4|>YmXnvu?SsZAX9ee)+SI}3Z#<)87^oV5$ z9^HB96$r`c+1WTaNIZwea8pc>tg-Zm5Nb+z=5Q}{&6_6Vw+F6+n+n(x3zeTBB?2uB z$r^?)Wh85xYq;$^MXjv{Af^c;KpZjXBcWlvNeTcpJvTpJ61EywE^atz?G2DnN(h)1 z=YGN5@k%+O29NhvF)L28iGKMN>A%u*asOph=fxjCR^#Tdx^6fJ%bG^~d2984ux5LN zx!_3Z(tDfs@k)w-pkVe#26JLUOVd!Xa5J@{IA+*(k0I<$6@-*m;0Cl^{S{t1xJFw=9n?)Pr?5h1VkboW=Q<281$*3 z>V>?jnVFeZAZgqt01q!BB1}MlY3Dkp!5XppYI$*SWrlS}HIiX7y8S-XRxn|uAx+8A zpTfvVI3r{z8<}35j78}`ARHx-@J`-Y^MD6_7{2}cmHGJi%+L@g^>c~Yowc^McJA`m z#I55EtOmZfmBIUY3?jf;%bw%iKVcQ=x-1uzlzsy! z-fx0J#ax$2x{QJXX%W-PQ`mLwmltk^IQPY{1UfnnHN}9Rmqx@t3=Cu#F&2#f6_NGt zY@)Uyo`^f7fvmdC5M~sjBfzPup6zfgY8t<*g5X1XR`K^`PsO7MpS(ENEtLj}rX==_ zMY`W(r-{*^J$C{Fqkw;q{uK^C$^FQ1-2G==S2htu(fiOYyG9p^un{UvGmxa9t@}4M zrWlj!t)6nFlx2(v*oE94O{ROr#Iy+u4$je4CatqOdr0O|O1)lIp7?(N*BX&66p}@d zcSvOj#Ao@Jth$IZ~RBUUdOl4?Nc>pvAXI79wIs-~Uxj(y*{)Wl&D zPhA1i*Z)cz_CKqF{vvO9u?tk^x;T1zdd8rt)<$*D%ODig%*0?EAjQ7FRvv<*jZ6`_ z%O!u6S#`kSfCD>(+l4I3ziZcNFgFaS4_>;KC_D&DNX*5EUE&jP{_M$tf~V)7X{`TV z%J5RNM&55>qUAFOGQj7j6J`zrngQ_yLigpP#v0`3L9$Kg?7R%p?TvjY7MStlKTue? zwruI2azkPw{Awhy43<);tV2zryt{Yz!y9GA87kr8@VIs@ux1E&_x(Rc8qYFu*Qu8f z5g{UsqY(zc>c#41zk+ov9&}8Pml+eL41r#}lS016V) zH^D=@A6ArF#C)pbuzo50V@sAS3EFq@#7!sTvfu}HpV#vGN=!T=%?r986Y$A2yZWCh zs;W-H$|G#vO8YuCW?$bhbm!0EjXD65?yaAH_HyfxLE|i!qjBNFv%itRk&%(6AWi^J zQ3PG1oFpYBzj+nR;cKTwpo1oAcP%Jjm-;0Ye?Um`grLvA1R;9^DgPRDq0@c91Bb4d z_mQi0O3)2Hv)9o6@9Uc7Usf((9)m2<0Hms#f;y%KOI2Vkv2ZBB!m$aJ&D*!9h*^?+ zsMf^=$9IK=#IHCyo>QTL$_8M%ZT8o#qAFk@iD2WiRrGoO{GcT+LBgHyw`nWrA*6KO zEM|q;j z{_y5acuvV4%|Fa*R;xCl3;ah;w*N+@^fI_^&>-JM& zX&#`kJj?O}eEX5CSO-Zig@$P!HtRaiWO$afqEc3;=h+d*y3Cm~;@(JWir}s^V5_c% zF&Bc(9Ic;B7W5%pUcY&RnRyRk=1zuugZuE&$d)eC4)3d|i4ID7SHc#(zyJr;4NlE!; ztT?}i13sp7usHE>99I%ADf7cl-}A`Q=^$5GpGsZ-dLhmr-@O)UrZKUb5^H_}vao(v ztZ+FJ`Bh?oArAj|_&SQn#<6p8SpXLUY~BHr2(n21{_>JW*t(#=Q?Hrowkt7!PtYpiu-i> z9(IQ;7HCz2*@z_K#)ExhCa?xhOmoq^gF(Ht}-B_ghU+NT<{p*1VA4(hug|v0)N@jAqzs zKVFs#q92NilF;JELnIY_53xM{>sPCORF3@n7IFELPzgU>ke;5*U%ZBJtP^ven^KRN zwwZ@MMCKVlsSa#PX&lr@w`eBXD0@~pKBsx^SedwH%{?{cOCFud`uK6CXFu*|gQ<;U1CTu52H}bKVU^L6wyW08N*I9|y!O)(Zl0vqx z@KI=JC}y+{qxp|3m_x26x*ghIpQH~T&VqjH=}*kh;}ub%Tfac>a!oD8-BG1 zG0KCkA;j`Yk#ldNnav6f=^z{Etyld0OVLnJ`OUv zgG^?ISB|j6a-%^U6fI(lMaJ)@Iq2$ru6fwCzrDv|E1(|BgZT%T+b!t=scWXDr(+Ro7)%VDSO09Wr_U&(1Aqks-7hq*$ z%XfYkpcE*i=anwf|L+dAnBR=ZL@Nr#szbA zW21l|q){Lw&521-tF{{u4~Q3S5SsPz@iOzo8n8o-AX{^BJ=5CTGtOPd>jUJs!5%jpD=jpky*5#BD;0C|%GjKM`{J$DOC6 zp(GipAEvXEQ37t=KQg^I{m!P@p24?!cT~l+XYNa{py9H-=`UO4A;ba_H8IG?dNqG) z zg)W-Fp$A2F`4>&U4rdtj=I4h!7?7E_NfpYF69D!TG7L<;&YQbz3=r<5IgV2kEM_&P zlpM!|8n0ePlhep#rDy(8sEDhdhCX?sbKw}cei?)V_VH35F`E6*6>eyu{#V1EqEH#C1px{2c5Kz`eH~f~QwQ9ft-Rfm7(pN`7dla)-q*IB5bTKLi7UIj%>9j< z^@^8Ub#n$BM?0xeBba;Q-?JyWU=9t=hDT2@=`ub(Zr+|8%|i-K+yfP4*|f_=v`dTr z)7>SqU_k>Q1vcuc@F@TSSKod{<{#h+S^h=ib_c&+MileIgPk}f25sA@)+7^qP z6XWA{xAX7Ag~UVktaVS|X52q~tsw%14HUd#nr&`w?zYdv&)Y+GW(Xs#nPB_q`3ufb z<&q`f=O$Y{647&UJZgoyM}z^?&G?L54sdMNrX6Y3TiOddXWr);&G$#))bzf`F zQvdPC9~D9!i3J@rcJ_SIJdu$5ktE=mu>JKKob`mo!BbX$oc#J#39{z4!pZ08k9u$u z9i4JDq0*d$utG+Gp;NHVAMEo5c0qp9YD*CYZQuVL>yR0abOtp;-=SCNyr}%+#^c(O zbPvDaVBI!W8D*&EB1XjCE^!`uNVZnq7=3f zA35qcjBWYM%8{8~d{zRkGk+(%q4o7uMfo-^3XI{J5|hDDR=*JOo&l3o%*quWM#ed zT}NcxAVne(T=x{Cdysp@F@3xl>nbiQ3*aGS3~4hF=6SP2;`5k=BL7A!w0bVGe~6T) zaMqiEDi*u`^_k?{*OR%J3nbGlh6=DfA?I!NMay>aT2qEHQEy>!7>B#g92o^D&*!^Q zc7RwEFcL^+h@uhXJ`78IxSbUE!2~c+Q|XzKLp&||E`uW1V3i^X8UF5AY=jAVv#z?L zNaAZEiYIJHXlkH(?6m6-|RrcwZA!BO@F1=2vHdGpzQS7_!I@#>l&%HQx zS8}RtDVYSs@UHW8S0KqldL96VxT}7iqYGO(RWZ=1L#FfW3%5C*2V}qGavN9f$MpmK zOokcbF_%W>Pkp9wGRgP|MlgTAhO(IqxHf$%<&l?DESv<}CK*`ubVpTN)R53sw1#>??=vk7j6I>gi4b6 zYpv(Q;p;I6OW4e~UF-p)@{&}+?=|SUrp*|{@(G2xC@ibhR5Vh7rB@!>eWsb)*^#Cs94#bqK><+6l&0aL zlmUwZkcF_{I6y+0IS#jw)K)b)hmkraXrWkCDCFRQB1}vKh?f?9_)vkF-3?H{36}Y5 z>O(aWR$2s4Msz~*B6x~3h80PLRXBlF^glKZQHIZ}8VhU2)HTVYaZzj()_0qwDHi?k zXjBu8_xD>X&Ec&iZ4PmI6EPU_X#fvtOQq(=A415DLrNhWR|QHExH8W77`2jFLBKl$ zleKneYrowYXw?zOWIaCOLlt;cqTs?xDvb(&OuGaK8_*$Bk_lb3)~Gs&KN-F7A=662 z;PzwYtrin6PxHFbjz^4i6p|K(Q^GkF$>s&IqZ11p@h_mTCp_08rd`Q%iSkVB5V+)f zQ4d4#9FCP>P(ET3t}OtOm}>HA2kz0gW$@sZC@d^2K!OV50ss}}K!qcqANb9(ikBgC zmhXMJ4K6ez@cW3T5G~zW}-(8AfA{GNLIuNsw0wkNiPwr z66u8uRWd%k?1mHwW-GaD$6?}aAz=#e5q~dyCmv6r(r`H4-+9o~K09i}K(( zc3muQ7!g3s6XsxE6`0rwZH9qIE>zqXXn*+l`DHz38(WDvfXKqQ?bYiQk6H67Zlma~ z!&hSH0ImXgh#A$>LW+2A_(-wy3OWbM%tZmad~&M9H@7>~^*P>24K|D{^svliCZhyvR+LD&p>MUr?C4Jyn^|P-Dy=Ch&*tp zKZCS*hIWdIFIXVo*gzZ%q`icJ1E%RH)`7sFAbu)og`}l#Adv_@DcuD(Sz++~uwh@O zBC2zQ&GK3dA!U6ttC7Bn5V6DuN__D|k_7^pqK%e0CQh+3y3VnWl3tzTWQ&5f2W1^d z5?fa%7Z-EP-1;~&7XYEMRI2rWBigP(j6;*LnU`O1hlrPT9`(E*PqR&lpfemR_Ard} z;jsMeKCc_ZcPqqT^NvoW)L(7Dcr;37Lfv9M`7}Begy%y;@+Bf2$>ZxIE^p$v1GIwkP@)QEI?Ov6utSG=ZC+h;T6iQPyA;&3&SSbs0z*Ogg517GRx;no9fs|?ZngI>O7cZ6cJ+KXt z??FLj$PbeO8TWvlF%o-V@Xna?1{KB@x)#UMHtwYyl(xzY;)eM<<@z`NHFyc!9{*^p ccx-V=n((Fo?ZOK3rA!p6jKV4Ui3>mfAAB_zasU7T diff --git a/codes/QLearning/results/20210312-165244/ma_rewards_train.npy b/codes/QLearning/results/20210312-165244/ma_rewards_train.npy new file mode 100644 index 0000000000000000000000000000000000000000..489cfb4168ab981b120a33863adb70ab3d44d805 GIT binary patch literal 1728 zcmbWt`9Bm01IKagE0V2E?&A@O9h)KRoEbA??r~(yYiwXGHV}gAT1$^8_=4NKZuZVg__5L*lnNI{Iu=wQi zRj5?+$G%%Ggi;E6N}Td>n^hGuq9uV^^OSMd=q2#4og!GS=HknMrOYnb98kY#SMBXO zkE@#9u&Y0dxW8`rkY%$WQXed9_&yVP)r_YDZ__as5`J*gLMjX@Ro^_?dJ5YGK8r)= zPeR0dEhdg1M|<7)=PCNhSkpc3Ih=bGf4jUnapc2yxZ>N^<~p4K8r|1a!#W<{Zkyj% zlW`ciz2y&!Zp7fGob-DFH5ztg3BN@*3gt3NH{6mUaZ%<;M6!1{ZZ+iAE_8%q?=K?R z*cTz_7w7d041=-pR(Ph*NFekft=>Tjz|AA((=Rpr;Z0eQEq(6?o!*uIjwt)#_^mOq zy^$}@U?R1`(FbcN$TG1wfO?bc?!%VfAnlyLRnc29h$kK=ZEo3%q0PoEhK_rYkR7u> zBhnjJzt(oU=;?*HBO3K2D=#!Xl#(a=JkUC?Vd*mG4&xX#>CNP?6zBm!k3^KOe6g4#wE~)3SRW^uFoESc6 z{7{HE{Z&RRM~MCzX>ssQAu5N~(qEOkK$~dZnUx_xJ)ugomM_5UptkzW1_2_O_f)bY zoKd%7hFj<1jM+3-L-M+h{r%;V$uqk#(IIdAV!;WPlj$*`K~C^HX`;;+I^nd@@S2^y z6VzJ|1jwy(f^@00Ej8Z}^$DY$Bc4te&~?y~iXCBR{e$|mc1OthtNWesazGfv{*T8e z9ndMq+-JPc9$IM>IReifPD%x-ndN-^c$>a#D$hrQgY1$0d>(!=nt53q$ivVhxzhcG zJX9LVaor}l==Iza^=yiV;;Fpk-t|1(v5D9;Tg1aH+-KG3jhX3Kf zMycG>-J1i+ZN8O75gU6{jOw-wz%*NKz zWf}Jc7PhpH9^d_cHWqt!Ra}r|V@nZdyideN+nd+*15_3|nRWw)PnZatBSp)nGqJDi zip6$U7RWpW`}K4df}*Jv_nVk_7ueWsThB!NR|b7n;Y_4@g;S2IGI71H>-i2n7PRJq zbY=^gn0Unw`aO_|=bo>JvkMt$oQm?%U@$@7CX-b|W+GHzv8q$SzzWec&vFM7)#7)q zHM&eRrtS=A>SMq&;m?ZJP6kfu{%F?Tz<|zGiD!459fE#~44(Ry4%IqIm$n%L9kM%m zpRH%$@3KGaY$EMoZW4E^?k^fzR+v8T=nN=!m3YL5f4Hx zxE!OSyNny6;zfhi)A*nHB zZ9x=TdhE%iz{uWDtEY*A(~n!#L&|MYd&k)Cy00x{zg~=Trc-dUUEIK(A;VkpApNu+ z1;XY-ChF4^Bwo`PRv}a19^ml4U6YKUFMjK|TWgD@O{#%SBnn1aM~9-WkYSpWG`hQm zj602)yV@0PF_CpCzRrjOOS9sc2am|uwLDy`u!VweMmx@Do}oa#JK4%FnF6+1Romc$f+j~Xg#>A)n!?+|Hqc1oH=h%d zpyt7NrzIsJ(Yg3Z-I@)Wxr$?98YJv2Kigy4ZG+7pG*c9RvWDTM9@g7Z8`!43$hekc Njn9Y=E=r25@ec@CJfHvo literal 0 HcmV?d00001 diff --git a/codes/QLearning/results/20210312-165244/rewards_curve_train.png b/codes/QLearning/results/20210312-165244/rewards_curve_train.png new file mode 100644 index 0000000000000000000000000000000000000000..3353dee1d3603b2fbc38372b6c038635fefe1440 GIT binary patch literal 34560 zcmd>m^;gtg^zMMtCDL6AD%~vt0xAM3N_Xebozjhnbg8I_bT>nHcMlyBLkI&5anHPK zeeW-K-TMz**HUJf_?+2i?`J>Hv(Fg$MpcOjpB5hife^iX@%$YGf*A;bU@YTegWm{` zF06we;;sr>uIdgJt{%qD<`5NQS4TSsS37GHW_NRE7i$NoAdeuA5ErwRtE;1n1TU}s z|9%3GgR>>?z4rt4;7#xxUue5PASA}+N%_lB{}mYl{!Gv%r$YnLx8Cz&JtHM0P4HI4 zc=P=E^B>u7A>`nGz9l(C0o?yf^Z(Ps2{xwb7Am6OLY~>FGmq@Pt9cB0l~v}lrMkMd zcCnosZ&Ux+AKdghsN1rsr>R<+$X9>)>eYA65(D4kUJmo6lG4&o(}n?A%DhcqIy#=; zCnKxd{vMqwXi4nl<&`v*y_M|~cYf{(lhLnt{bmUBU+~^7vYRPgsU1R&Dlw5VF%1u8 zNn)f*c?rme6BkvUG46}^U!EMcliUeMd4F$49Z0gPq`OX_UZ)9!^61x>2Hf5VAY8^( zMEPB}-z-2y!K=)V*Lydh?Fq;nep77Trhj8^nFMVn9)VTYtwwS7H4ca1kwwJE^Hy;a zQ1Pr+&qGa@TR#5Vn@y}3l8EkR3djt`AxSP6#KOXAJnrMv^F8YPcd(f8uu(~ZaAexmE*;UXnnY%pQO_TN!Nzr$wq0)22*q^K7 zJoVSs)?RRICo0r$NaieQ{Cu?5ZAy*ZpUk~>v6Zv67B6ElRj8eG>EDPtXlM>-(BpAO zEDesU$qe=+u%$_Q@RXI6QAi-TYZlssr}fme2c`b(jOV4y4JpRbVmR*nOc8KgZXxr} z)->?iNJZA2uSR>_UL(;jH0_Fc>~yu<8hmwiGQ8k>GN{Ex`AaQF)zj0nZr*YII>&31 z2HLP+ZTfV`r0j4jC!pK|v2@>1rmn8;3WWXVQ|Mf!^+2|ChH-nC6Wk9;%D^yK@3xyN zY)feva4qc6c!aHzA;Nv)MM5vuayo4|oGtA`Eo?(B?ftK1-~FVWM264n*esb_w~H-a zD#)06!xyVBPYRJV0mTnw{ZI%JM|d$rhgVvfA#kNVd^}GrM*!ggu2W{*j=kiy#USBv zu-9C|B-=Gx;+=co5Rl^8z-%HI&s~s9jNuT=KbQ9ESx2s7jx{=c(Uj zy*Vuj;P<=yCy?U-F0!}SkOG#UB4R&x4rP+^}ns_`i_D^5FJcl>?)!p8pye@b0lnFhWz)eD?rB;vv2$B|%X1-$=R8Ok)dA1w4rBNy5q=Vi)G7wf-NRow)yVBzK- z^vL-pAAUSz8qW`&>HTlJ)epJYz1kU#UL;tpp2wmay3{c4VDq{CeGhY|WWwY@n3h*m zpw|N4hnW7k&8A^;^As_GHj1_3|@-&~DP;zO#vC(aeqt8wQAd#yS zzxfe|k(r-A!DUE1+_t1?3i(lY0bM7ct`Ml(a`XP$%fAIXvlSdKU%q^uE)+R2F_BwM zW3<%hg^pNA$X#qg>XvIHTuo6iti&%@F&5M4-}b0EaWr(M^kYf$WfYx5<0$WySC##I z^1K5q3uH>U)hKI=;N7{O%2XPKTCE!?hN{SeMH&IK9{%vV@jrEIppd<}s(z38KOYLH zCEdBeRIERsHfx!q63-|twiYAk32K;6B@v5&@*coIUf9*D zN1S@(}6UfuM!N&-1>r0%ii8x*X>DrC7W}&Y>0opY zAVHH~`cuD4eprp1yuA47gnF7xfPbXv$>!i%VZ~6J#YstiBfu5c?U9e8xk@RXuQNpK zYXMrleDUIu3$+Kx^Q+^2-WUeSDG;n!8o{sxHl_Qtw0|4ycc+SCm}IiSo{xZK1H{@u z-uUG8`Ce?Kpnngx;PdCtsRHKs0CS%0Ad--l(~^kB_D?MGFJ8Z12i*i~8T_;Hr=a*d zz@~q@)A5mH9Oah7*}2Wx?@q`eyPtdx>cUB6u0A=fb@5vc(H}G+4Mgl_S~?=h_*D`C zie`0$9U9#{Sv}l#r11SjcV(tob=^rnn3H zR+b0j@KY3*WtJPI?R2sEKq~)pC8d*p<7y`WKLAoO0URAlG4#jt+_a2klum~S+-3#b zoa9`g;I|e4^#{@fyAcb#jfa@G$befVXJ-LH%OSJ%o`ivP;pkfw?3NR`n~_L z3b3^E_8NYBw4SK8>fS&j?)(767n7NnH{Q0mE(}2IQOg~Y!!c!Er$OOqK92*vZ2wDZ z*h$mHW`-+bDT_wX!m;;Z$~hDUGGKeLK@?nSeZghe{A_Qw5tNpa{d^gtOFwtbA;{L_ z_<*b6yd3}Z%uM=h(6@85$2Xjv7Tu)GzEcJm3>btK0$RV~j zTM>SHjR<(1DV9LMQyU1oxs0}-uf8})OiSxklezA|qSAwSw%w-ko8l4^Z^CY`d2g?_ zWVFl8@TT=#h%PQJ%%4uj&*=gTF@GBi{xx#S@iz>O!o?e;utx z^Md~Qj8+sn0HX32nX2~mA_O4#2mrs_?D>87HCKu28}i$qMM8%D7Y~h%jiqj{PV@q< zPq_@6GttQ>I4m`;d8$)sMM}n@dUqK>(pTja9@m|*P;@gW22J1V0K^QEG1vuK?12Xg zL|41Pojb+A+d4m9QiCpTouL`F{#p4d?6}`~NPO!&>|+x^#fzQ195LjA%k@D+z^(sO zv3@`3ur{ni!W9K2B|-B(ax_{>xbJxmOs`LB8a|Yk&Z>d?XKW5;pf1tz zJ{%AnKHRGqE_d72NEfnh1B^@+urt4X?J^UENTyVD8wbSW_!_R;TKjKI*{{S9)HB>4 z#Z5rTCH6PuC|BA7^fxYp##A)LMd!oxfy=No^0)B{$SIS~D2gjY!0ky>_B9G=*&WBg zrBk{6TRu<8St3gc5Uplo*usJyqxUuo8qp0KKylgX@uVPv5xpEaKR}r^9<~u^6=^`c z&-Qu`SK2QCHv=GH7P-~i)3c1`kbn=Igh~afW=ZfnufKq=Nw{u9o`n-L^wGDS@MIr0 zPydXJiei28q_0H4GM&9SH2?R!KArCE?PcNOin+S-Z3d0PnMgJVyv;+XBl2SO z#4YOk_{78(3$;$o^WSrGwX19>ep!!wlgEKCG?=PHTTT2X1q6LFnzz~L>inpAUglz* z%*4c`kMMba3a^X}T^ym#k~={1AA0J-*d&avU%ZHb^Bs!9edXK3h?GO@9I*DRCo_P68p~b6V}l zZPwJ(ccTDrbw>&&26|wux~h05g6ch@rxl=A2Vh0&WkbqI>R6?5j$-tUAXU&v_2b8n zQa~Eegcjbh-qt49a*}a+wm+|3=lpP4d*_3e);8*^POT#g;Q6lmbCa5Qj&*v3Gy)-$ z1)8X^jTS+N-?aq1ji0nDEZ)9<-vQ9$;%`BT=h>{KQ=J|;1qDX-s~6~WT_ItXlFDR} z4?DlP1$6jRPR=l(f7eg$FEL#%8Hbx{gG+E^x!!kja%y$~N&uUPw!;B-y}8@2>(HbC z>S?y#jZ^04oDJxv`x&ARXH6F7r_-D)EUk!Jm2{yaqMu(jtm3r+&bc=%@{;RvQ5u%= z_5+4eHox!B6h|2y>apaNrVyOw;I7352n?or8nJCn$iUK$>Pjv%f zWn%;U>_y$>Il$jlR|!AZJx>(?Ub7XJce0JgLG3kPY|^!my>@nSsRN}J?{x#7n2pW^ z#BkZ4ebCNNLb(E){ZOA%Qo8H5viku__JZCh2J*Ldzk0r}(2?O{jps2a zoK&M|kyr<7-Yn~8L?dHJXy|C2izR&F9sgdoC(TQ+5Ev?pg zaa^T*sG9)-qs}1y-#^!C`*{ru!{g&)Ts*uLKrk#e2hx7z<&FP-%LWvP;)@r!K>qq8 zWy)a&t*x#8w>4T^-3#d}R`D;DmDwaD<}?*fpX|GF@<>ZJO>}>FuBD|FInPtOjQJF` zCkT3lGUyLl0Qcji�I4-_$KaEe6v&fmV7YCGzOe?}@FW8@UyIdmTOSyjP7lN^3<| zV803`F5WHE6f3K!aKC*hHHYH27@&rcl!H!X5OD1Vzm3-Xu;(rb3eF$#K|_P`mv#0V zrY1NU5v|aplczZK%?nAlzpe{Of7oD(_wV2LJnE$K+DOv$ z1Vq*g+_nbLB)aID=s>)cFeJ7{2(#H!pwjOEl#r=K{WK2r#FJr}JiuGgM44X9@$0|} zJgR!dGt2L6cENq#9<3d}er2ZQ(d%LGSa`eY4_IF(A0h09;V~QA9gzHgk#zJo=|J(; zl;HdS`0|AuFqRl&YBMvlFBfhW>Fl$!v(58za&lJ(>e)HE%eO8gC^zjX7e;Zo4rVl_ z5kr|D${pQH65w;rvk`82SV~1({Rq4_3jeryIcJAvofb@ooav3tB?V-OD`2#sJIY5o&PoRJy zB3iq6u%^R*F(OZL@j*52s`Ox~3H6qViInuCF*SOdI?6z`kufm%4TyNcINqtNUmN5B z1zrK7>5V*yU4(7{0=b!Ud&L1Ly9rEdZA#O=q<6PcI({rmT6hL%erNl2>g<3s=bN(@wFWii&eVsk$fXf!j4EFEfI zi8i2L-Q#nki)g5kOvF03!x?UDfb=QwhnuhN=K(e4|ww z_g|@~oDu~;c=!-)9m4y}4&M_;hfjc4n$*k#9e!jRJZOQoMUDqgC6Yl!R@=?W%~#uB z51oK+aomc-WK^Z2-Rvu#kdT0;+0BP3iHYrKA#An1`5e6MimSv0AcKB2X}@{_kMo)6 zKHV7)|NfoT|8j=|*1ZU#h$c*dmbFdiQ7;04GHcS?FV`u`1nmiJYk>rSBm9U-Nb&$p zw_9qQMN}Qkl(7N6rkBR~)aPt3vVCP4P>jO{zu9v0y``p%j*brJDk3TQ&w$?Xpfz2A zW(kjzFJN+L5&$&pB@z*!x5d4bSANFKrEF#OXtCaHt)Cb6j{pw9k4vj`Ih8;N0(APv zMJR9C^a7BWXeHzaPeBVz78Vw%^W|Uw+7~Kpzir}WZo~lL;{Rn0aHb6?VPBU4;0O}E z`7Xi0#DqZ3FP%NG9hyLj18UcQ^|RWvJC2Z6C<2fRU>n8V?=}S3Fg7;ktL}LT znlcd9%bGhbb$SD!e;)vh5_dI7KW^j!r1YVPNOcDi0R;6qfVTKul;)qDCxLFyfhSZp zYX_)%5>TUYfbKlK+-8GQGIth8Wl6Xl5Bf<0`gpw=jdox}mpp@kZ<7q_F%qE0G4NRU zZKt@K-Rj^1P_$8?_~bpDbV}S#u3`y0>84FDx_iZJ;VT3 zhvZ+WUtm{&)!4G$n^?CNCuucX{xHY?@-aY%)dVHRqeV+}9|kViRFUopaJ?^z_N0^1 zr6%sOX;h~5itrTh#H<)P(Kz69I0KQk=3Iw%&^a{m&k57F^M@P{Qqt*D#zIzxU zO&bt>UYlu_fLq+5(szAZH;;V?6kfRl{3$w+#r1!mfYk=i6LGw75$W zcIC#SLE!wN&8euUs8nExgH<=TFwoHSG6h_< zwuKVxjQ#BAhjq_Y+bf%xe0TWMTDIAmoJXV-6%(@#oK8#d_&?xs>$zYJi*M0G8;=yZ zD5D*|>1ZQC%Yb_Vv;^V|3s7-3A)zlxoSMhrl4M}dbK{?SPi%Q_e+x(ZJ7^mo?LPqI zeZavHesQuT2;392mk<9QGY@Tu1zQt|_E>;*gW6!w4wt-lxh;mX zRo=fR2byaEq!O57ec2U`jE-K*@>p5{i(Ll@`{V1^K;Us4mIPe)Yrd=nDs86Dg@eie zoW-H(loAL8G&|P=KEoCK_L>Rg?sq@b*@hq8lmEsNXi@{LqnDYg9C=ypwCJ6G%lpxy zN+%5r27vXjmS&mF0?aGmKLReeMt4~cjB_WTb#KnsX6!_O;-&{#=Q%-AwhA2n*LkX0 zFO-!}=;p%P3lzZ&VCej44M++gg_3~5cJZ5&>-H!1CmL@s)amm15XRT?^g(UjchFyPbOFBk-XG5|(Ax__HvKkQ7XtC^#R zBEa6O*+yHEjc0SV&EML9;aLvYI<$5L`vz2|3;KOd8dLO~@z9J1J(TIsLae$PK@sfM&%_IPKt-ppL+L#(m`hm>duXV== z{EYYj+PrcTJc5)Bmw;d`l*(XvO4s2b(2W4doI$Q@oIeqJCrzlX>62_z3C1uY4od@I zJ9m&7XAe{K3@P%#J){PrQUU?ZBP`x zr+dVn-2}Ns!fwkf25CW}g6^?3XIsKu22YNd1_^cn#91BS;@3Q}?5yu-|7XlQ-}2vl zSP;yoz@O6?%&*}73+Y_QBXIZK@FPeRxIZB{hjVy0?N{WyzA!;PH)U?$5CiUC@7u2j z$g*N2$wYVk*Z+P*Wfn-KTo?3{L)1Ra+kZ|^SKryHn&s-~2uVmwiyq%8{&zB6S-&Ln z8|7EHdwu7$jOX`%KUisnaNBK);23Y@TmGNR_`j@jq5-n)-rxlc0va(#<|gmc)i64! z5!fH$MK1KP+eRIMxeQX!bGN9*b3H-H6YM*f5TVHy!2K!Laex`m|0p!!2Ld$>GCdp& z9R$H(2o&o8dMpV<$i?MuiEX_XdUccE4X|2(YxFrWQQq0P98nEMX6S7I7LaYJ`G3#I zGtejS1Ogo*WIs27I0Fm>755=zP~$>;&Zg@8T7P=t^hS0OVXBW{`W^2k``?du{HISK zn;RP#%Z_4KP!fE6pyG4owkovTj2vU#ZLCzV;yLB{U!;}RwA%UmeZ9OU$dB|ALh%ycrt|7ysw5WcWMrfvo$qc$ zs3i2)*To|vaV0kWT#ln=szsw!i@ujJ{0ZaMu85x~#Ju!~Ws?|iiV=LD6)P#&!|gbh z>(_q2yWE)ouUyw@rpUhrv5+zt&*?r6Q>Nd~P{KIu4!gFSOKa7>)*&VRyQ5|fi%&>k zIc-kYD#85fvF!ooc$GGjeI6*_O@Og>$)Q))&>z&dRp&ZR*PAw0%}9sJXYWUECfVJZ z;?W<`&Cj)NQR?&nU%(lkM)K;xoQ8A6lOZ7$f9s^4iiA zmsipALTMMhHpxZvB+sTr`1J&8?-XS`|8ltiD}fJ(%Rt%LStBA-kjDfscB}J_TH`aU z7^qr1^QyOuES}!_1z_c;VP!7mVQFShm9?=ro*ca1 zlme^y;iQA0{8^3iw42Z9#32wlo}LoY5h1pmNIR;urR-gr(0lz=yM5%oADEPF?4F&R zdCx=$e2X+M)Tv=BOfZmax+r=}OYx%0*j!^{3B%K=P99YQOM4%05q zHSC#-FkuzikW1-dNw~e)_yM7}yVdh|TU{L#k_MUcLfRGFvG8exrJ)TEj=*u54tEIl zycIoz+_VT$1fff*@3p_k!Hc!bb&-FFxe!?Ouj7G+0HbthSLS#UhIIPH&;{!9y76$< zFhs0SPvXa&>-vr|HwQ|Y>*ipY+a=YAw>^K7cI&SHn2l8&d!FR}oO#V*t68|Zpsw~^ z7pdCpG8vvDI@!smGG4f&_&x=2XHZn=Q70PvA!%u8g!E#c(6bHT4KX3b^?5||^Tm@E z-`j~0@ber0__;xkq(V(ASD)Mjsl2W&+$t#CXu)6 z&MqK0Wz^U_H5tOX;}H%Nv5Okwuupq)IY83>bgIm=xkZ1H(|vEn?9s@OxU$q=z3~AD zU3amOHtqw<%OWZo4yqf7qI|o>jLtMa-On?Rr%K{Erw%rzKlq5{swsbOm;R8x|5#g# zFqks7yBNm0kd*uslf&e zbo>sZBL3QSIDy93I!?2nh?}1tyc#t9OV`gA=u>i_65P|y_5fPb8ba(C8hR%`+3 z?SI)%`SjYHKqLC2wYNd{CuYugxU0Yli~H+wv+?5BB8XnI&aOL!v&mstqU?Mt<#+3= zY+oddclyZKtu|~(`VCL>3E*1FK8^d%)y9RD%KpGDaxJF0h>wf1rKJPXes~4v>ltz> zGZ~k)6X7KKE@w4a$ilKhYPL>(bj}Gg+O#CIdCWZVsH1YG?(%ABor?VEAcfQM&E01x ziv%q$>{%~`=i!puGpy;qh~)VROLgpL4uB~0SzOjDK zg8Uw|w>yfCfSs7Lc9DlG^?CT56%CbYv_)(wc6fs3wn^__=^sCTp|3gfsIBG1iJsEz zr}2BfSlZ|J?=ucHS4vGeEUEB?-lYo_Bjnhzc+iySU2d(3Em@6VUC;>`5o-V_iF^aK>mu^4^!_|Y^PoJE=i6c+7Zxi?(b z4pT*79L|zx9B*~q(et&D4CAQDM_0l#Jf)o5>ZH^8@!Fj>Xb2V)AlsBF^Me3zMt}No z;qB8+ykI)P&wTht3PPZX*eLipUw2#?*HIIde|W)78Ch;h{H27Twvh2&Yw3W7l<;gpTMijjV-Z>8$tv0;W?>2%m)G1cnu`$ zMm5^CsJ^?u0%||+Enw;>==Sstu=Y5v@ge)*s|+UA}|wm~fOq1d)>CBP<1;@+=W*&pQm>|K30f%}#kB z^?E$n-hUsXtqX8iXJ@{K7k&H&%$LQ^f|aEWm$6!u40XFU7t=C*oNW~eZ47tp3kW~n zgAhdPEu6t~oaz_!JYVS!$^5jhV`_l<&^i&x`AvRZ#BtUF4W<=uuvCZZs@Y#=Tt+9f z9l8-fhHcD!R#04n@2UY$&P<#5Qs3g7|=3NJU);%Lsm-nw?{8{Xie^)tIjcLRFi+x^q<{U>`+y^eNcqwm^FrS?$ z8ZS7%6-93`PtBInz{Q=pk)JAe46d$t-w6`DlHQtG;o$$;VELyj-QHU2VkoCpCGlXN zR$!$(+xL8peUoPay9EIKSh4m4unYo8)!~z_u*4+qPcPWQnR9WYjU9JB5vNiYF8O{M zoJX$k5jf4tQTxp@AM0yaG+%R)D)QiQ+ogJ??G5*ITX^u*dT`3g2VWRpxZ?||;|b$C zrfYov-qzFK#~R{GA4bNRT>?*?3G3(*G4psui$n5*%#57Wj(nvf>R}^B!rO$sxw;Ho zHYN@YJuN)ZV=J@XE}Mxa{3E;y9Mii>pA5GjaFpC@2Q<#%dbs23IZUN6Sss<12HgwW zjc^Gy0#g%nB8ywuVINCOEJfV1vQaU8c7V13b43b5ujHt}?OMI;68(Y|_K8Q#9gi>v zf-?TbvG!O z!PSaP$@9ICm>eHruv}h-K92Kj@@EAbo8*!|l=HPLO${||wDet3+%$!^whV&|Y2`f(3v-4c*7y~yRm#t2TlPiuHUk`g;y?myMH@b2_FDfc? zhA<);@4HtnpN^1qAonBO_DzWHy|6+5Z3i5!ed=oF(pi3Pk>*Q#GT3!l=*Wl>w5d@h zj#+ZJsUc=!f-oqCHHeTSQa|10vNx-AhJJlrvbV==tWuVmGGlPuAv#GIH0(Ven5l=> z#n$$Xzx>RbiyRvKz*9bmM)fnoK%~?y9pFlSF3&OFGT&`MxA%z#P6#eZnc!)1BHokk z#$F=H@e3p!jEu%+>!9F1IF{25zskz=<(ST-7+wwm9zjExp@vu3Jd|#9QS?Wx^uy=? z*o7okC{H0}aCkTSXr=LNK~Hoh^y#HF8F5AufvWW`-F>OlgyDX};uS=Z_46?CRM!td z2r1~eSzlr2p2{hgYW;4ML32I?%}UR81I~vWs;B{vjKlymzE}KOm&~i5rOS>p)|7Ozx!|B|SkgqHoHM~9uwuY)&uXGje5yw) z=F=m`5Wd~qjrF`=_iV`Dg(N{4R;fn{9%?aU8R-_XLn_M)tw+r{!nV~v`S=~-!u>_A>y97} z0W}|E_T<`v>_I2@!9KNf*CUW_IZ@v7fFagV8x7l|-&j&eb{r?_=0K}GcbjAkVh|a6 zZ4+3n%D5sqNvz3B#b7%~r;9w_@yE zPl$Y-dPAUHmknP?tnY9dztDdPn84_CazeI##H7ixb4Eu^jK#u*zSl*+@Sa-#T6MI~ zNMiy=kGMY6!nZM9MwAg^v>-|Niq!kTa>rNiEQ!s;wHqwh-X=>$z2^KiDf=Z&c7EBF)n37DBV&e$YBage zkR^bNhfAxSE{HAU%LIK?CbSS65OdsE+&o;#d=$w&+rhT5XhM%i#zeHFOMH}U4?Z_y zQgMpz{rp{uOUJXIwM`CBpXn+qT&&NDohQCWPuaEXpXC@57TnqAP~cu+JkApoD;M>e zZgDMR^yEh}$kOI3w>u~g1pCY*)@j_0a+mur2YbwycD0VUS%wj%I^63_DtD(q(Z(I9}%5%+xZ>#E+P_l z^-S4ktie_tKAesWnCThjL^^7`Qy$uz|FaM+2hp&9+B1&u^3S)K6w5>v@2IaH5!@_v z+MY&9-Acb52Bp2jF4SM}VU=T((3Mc`1;ckKr0tde9q zB+-gDn9F(edzAZIVw;A!u?x*~^{L=s;1X0Pq^;rktI&m&iwdX2HV9~4dEjf;DKvjV zZiQX1Q%R&<%o_f%q#A2{OD&o?9olVRI8@KDvw&Ka!zXFdRBJ+UAPur27A-a=d8g65 zex{I8k4Kc?W~Ot_-g-}Eu(L(QM4ey@Nw~4~Goxpq&-_M`PHcDGbE;&flcu#{n{@Yq zBu9K(8|zdPiZg3Xciikm1R6i1wtJ@z0C%0huP^Z1Glm;W%|NzkvBI0IuxGy3IUMY7 z?zVPD(JH@*uySyaWGr^g2K#(U za5!8^QMNYfWtnyzsj|hH6N%nzFVO_Vx8a;P(?^k&n$mrL`1@Ns!d7CnhiyHaCet@| z;7R|u7SVCUUGD1Syen4-0{;XKP(}iV=yJiCQIAmbjtwe6V`|xENf$F(M&;= zO&8-_s}BZEv|t+k7&}1gXxFXNc&OyOV?q$Uw${e-y+!eaok~g}_7U!)N(Sdn?acy@)cNr{Vzc#%b%fso4_i!6O6SIlQ*Sut$K=@d9glaWXTLU0s;IBv zsm+a29z>wkbs3HYr)(CllyK)ao)%eR$mI;h&X$Wt`{maSoSO;Ld02o>B5?DW5%dkm zo5%JSMy>2oQF^Yiq0E{hG-*CwTc`WA(fFlls$rpj^ z^&?9<0WT<)Dq@2<+(UsKQ=w8D8`oa3l>SeM#MF;77A z8P04`=0cHmez$6fG4+D~OArg9=yPF?BD)qC-$+;rh5IdrHLAI}f# zFE0D?O$A(t1Inq$Kd3@pzwAHBa`1Uo4L2}PtIKYwv-V&IR3Rev>=rUK#A&it+^(se zR%1mM>@C73u+Z8XBX!0gQG@6EZ``8G&{+(!C!X3q54SfuwNQI;zF;OjGsSQDQmYBSe#xw;m%@XOLF!s{vs;bjv zNJNFr`X0@I_Dk-I!|p+F8FugPZj&-<|MrsQe6u+czu#_lqzY@>_t;}!|Fd7uSeSHK z%Jdxu-&jyB6SAdm?IsW@_jY+Aq8D>OCbvF?=vi#|C|j5IKtz?TBPq1gZM*5Xxw2C! zRVr8NImIvmyHS;Kq+JXt{m1^&3bu!2nzB5Uuk}csIy`oRj7dzkKI$bG&M2lPgwG3E zO;nRzy)+lD7#ITJ{#&Kf3g$f4MK;5n%taQ%G9=2gHYX^LX;}zMe$oodS{%*ru#Opo^}9J~mS- zm=(wU-6?||NbSz-;5zCs%i4hf|t8BLtNSdx6eEZqe~M585>qC?~kMs^NyX@`_Y$7L`A6kLolw44+I>Q|orGx6_vOHDI@At-uw;x3U`S zDxX?Elzh+Ss9i|?ZM2r#-5H-KNM0TnAFm77HsWDDUh>GJ(iuKG+3aOX+E7y3HBpDc z#Qs5DIMnDm&J{|xl=W4joy_G9b}?8uitxa3^WN42A8!LR1dPUeteBl8h1p8S^NXCT{MPK z2v0Z=4arFI5QFgC_!ve)YMm4Ff~E@O+2(|!*8`YJHh0Y>sr{YlZszvz0{tXK*E5L!KP3r-X-V@)}COvDZ^3(pOiJx;*-dU4i594vloN5q($B2mxL}MUsO|{W{phmmtcMak@eHvX*Mzu)Tk1xM zbi(?B+De7@W2&?*xk$^h@%6NX6t(0HQwz4MnC&zOroX(UP~N8Qmr*)puW8xQHGB8g z^31*$7$7E+ah(`R-zq6$z(9~6KhQ145Ucd_CWNvCQVdUsn-?N+P!NM^1X#RiM;HI1>Cu@}$_RrsgV2=)1dkzM5lRpdeRZ^%VU#=in4s~hkVy@W9DW{ksFuQO4OicNf35loc_}| zkP2p?Jm90hHva%=Vks%@*_f}Kwh#sbt#TK`Y6ps}8cb>$8xFymh@ZCate%btmiYiZ zRelnMi&s8+GlI1^ygmO}vstgdF?9ZgOr_hayyQ`IB{q2w0CG8mM%vqNf7U+9Ylb<_ z6zO?+D`le`DZt!mO;kUeMVuY=`VT5}W5o(Bu>0>>N1!#cx+i|os+#wfNo1R^`^nh!^`mAzIy;RuCbTMDdfUL)7J+1({#*4zQn}nANKG z09v3+gdlK9JTULfW@rm$kDh?gWG`Sh47M@RNwvfD=S2>&if;5*&jAB4JhvsK+telU zQgycV*DH(uB2P!{0g~WkrrlPFOrp#AHuY)8^Se23C#I)SjO_!%U3HcWDivpf zMTn<*r~u}z$$cY6f1=!0^;LqnmZc*zJNx@(UDc<1al9ZUqrtfp4Am66g3cB`LV;-R zzDHcnD5B})Zb0i+r{(7hxpW(bYvMK5Qtz$mDxxr#0UKL&t?T7JEc9fUsdy|?v@qB@ z;M;6apViFKd9?35Y|-ntYU;P;F;kQ|SCEEqHt4L`M!P$OhPF>{tOXrwFE#Zt*pozK ztkccz6Q|F=5ylme|tCOY6M0R6Va9c}pOe5`sjYn(ZK=v41k{4_o<7Y02 z+L9KnvDRLXW&2Y+3;fP1DjRpa&r)sgK#A~&wIXk;=EwbfZaO=3GFTuBa0s~oGbOmgJPz9(&P1G;7tuFszLC{7{s2sXCM0OVD0^+LyTpNuV z`jlxAdQ%a^B@HH3X9R^htjyKTd1O5NRkx+~oQ;b}*VulHl&w%Uc5oIPTg`uhA6T^j zN*G9z&k25?8Zj*+A*qJz#{_|ZegZ-uzct10pCnsjeZ0CA062DK6oIABD-k;PUt}&; zOyynt-zx2CBPUn$1sDegklX@z1ZAe;mG~a5Obwtb1*LHP8J zqTqkzHET+#v5i?o010@+RZTH4%;{bsvse+^2b#kbJoDj#q(Fj(6RDY*kH)29$Xq0@ z+dcS`bTH#ZVg3~D5stupN)9>kBam#9Co%l~{e#}!=`5G~kFD!~PhO7iVgCjgh(dL;+GcwB3J=K zKerKx)}5WihxL!+47N;mN8kwW8gLNfGz-&+d#&kMQ74&WoxX1S4zs|3?9I1t^KEie zswo+D(NdEz7qzp!Q7VR&{BQ+`icc<)|3@L=AIp>$mQo78|I| zzmsMF7Db2LU~fv-{orp(ei|#&<87JRj?W`>>c9n8feX=Zw#9@WIIDyg_>Fi0s^(Vw zwWkx{i+~Hh(+pRpvz4ca!~z{exdcKRChJ>3om{l2r1v@kvB2N?B(ODHgNiEWj#ic) zPIbcV?ryER2xhjuUeITr&bdE%)lhJlMI0Y&EIY@r`mAI*CiPzF=K!l#C9~A5MtO>s zE7lYQKj}Ajcw6kYyezNud`0>+6{MZmaMI*OiACXjwjwN%xa2dp0Y(h&$z9mrVn{kv z6iaz2&uY?_Po)%#bOI%}w>Uh44R{gj##gm|ogY0rXWV~ZAhBEIT_6pW*30wd4?zsO zzI|H+F`Nw#i*1j>s`?GC-Nq^kTx9r@yH@dI3$o%ysqU zTMqVu*Zq)kp=Det2RUGA+D z1VRlRrJR3@NU-)$L6i$1$z6Pew9oHWhze6pfAb1cg5FV|eLdb$w<3BR;ByE6g!xg- zrwVBI)DQKfrzPOjl>X0$pQ|EjK5tdIl~_*NKK~sI#GheHbR;fo&x)5V0lt3oGPS!I zL`sO{@R?#WqP}_Q+e+@y0oL}1B#LL_LXWl1DK7{tWQ`e zRtCRthpo)tt1nOcl@VkJKiu2L5xdk2$(^p78&`Y&eElwfQC21Fmi_Q2flJ2o(0$sm zB7J!X46^O&Ra~X(OUz785;z;k@j|Ds%7~zpRW!v2>-^yLN-q^H?F=C(W-${(PcX;L(&KbF}qK~IHC(aqX zN_ObsDgp$&0$fXdA*#I1|`gTkem|DzwOw}0TGS14!b(f@x)Cc9CbV>g|2(U^N zSjNw?r`&(C$oSzER_VD2UtRc~ifwxN#YV}4;tv(mcFOo9RS0y>^RUXpR z8bN9>8lU;6N$&oj?2f>kRH3oJOSdBoNV2goXmB)Ei$KjsJY;}gU3g$LR(BsQrn@E6 z17vhJzaa9XbjT+`Xy?>--JhAdU=y#b=~8vp9U+7#4NaBHn+%vkysim8+vSen`|(Pl z9||$4@0*reV#TRs8Qhd%P$D8^6kOR=1P;!WE!k)Z*9gzK~MFSnk zKVpnLrnzOBSHu|e_LMMu1Ftj4zvJS&Pe<{;?l^x&4!SN_H}J{j;z=V63i$`swlmQi zu{xWV?}V`*nM!gKJH+X8eU!E~IyMF*Tfgx_=YjL6!KV1D<`rHooJ|3j@&_^qzb z`eBKtYe;ZbwlZ#t4>5B~Ql8rnk321JH@o)--Sm$g<_*aUukZsueG@gYpQvx!0xdM* z7<`d1xrf*FqPU5U(hkYp5rJEkQcqe@gSqXtZob_3LO#*HSks+O2Q?tY{7SmO<3a?Aw=zm864Ca zUBdpR#P(QgP1%lakMt@6+h9`DNFoLS%68Is+{2t8zKr;R7`Pf8PKk%AX9%k9SM= z^whTJ{g}fSQ7(b)(c+&ax<{E=Q>Xq{XwWJWkS#t`X zr!YL0*wJp!T-dxrS9VzWb2tiAr@Nu`4Y60S!vmVR@YF{visZNpA#8f*$w zZTU1!D`maYa^WLKQ+g*}@R=9m|-WXdOaGN#C5d3p4j&iwIKcCO_& zS65EQxD4Zop_{)<=>=71z4^+#4k$+VZ5(&*q^A+aUmvC9G459USovV`nD&Zxr{FHv;(Oq7}4 zE>q7e1Zz3!ci0t?L9Z=-jr-z#wRZ?>MPNFn#S3lTU5kWf+eRL%ahII5wma4&S$jJ0tSOGen@gqupE-Hvv<9G`2Ox3 zmmn^D;NJSdq_wNcJu2MHU$#`;{VriD{4z3ecboNiF~#M7TsHmceG=BJf2XJ?;~j0s zmzo_0lyI_^FaKks+WfPpT{Blv8zs=jy?$degC$KiygZ#(ubasuHG@|^%Jy9e`sb_) z4PN0%J@ztI_{$Z7m-h2=u9y3ZKdM)+mZSbdi5`<^D4Kt5eBL&U`(gF`H*G({-Qg=8 zlRARW6d#J%O;l}s!5t4i z6p3*yQkeVf(b(e!l}YOiHDJZ`i~nFLbHaG{@}=(lw1U|sJ>X#ZQgZ&r4*7} zdEbuRgpel5_=7&vJ_jNwv z-<-Z{8Y;!4P|nCN?su$6()b>ft2QPQIYDl0$!#v!{Zey)Jj0GK@2#kQKGWbFx4Gz!7;*64pQ)V z$)_lqK_Tvn`#m>u+4tF&_x7^d|E$WNc<5K`Bg%va4Bl|xX{;Z;kVoyq7oSiL-fcFJ zi?TFgkim%(`f$*_+odXf0g5~3Z_U2vA`x_`n&l@yL3LTgp&PR738g!DFP(WN;eM88 z+-(>h{U%p(vJq1C=_eUgu_r6!=q8TVZ;p7d2s84=b-l{WDze=V4$Li@Go2k%g8uDDWw#+`Q{j^sCp|7_1&Fv z8!%Aok9E)Ee>5+#2~*t9oBvj+^>CHLe@}IvRx)nMzjTGU{`T!_AKacjMxW8SMCDM# za|C*pQ=si)*ZF7>QwhrEW;39<+6H2w{eNviNh>I;z#u>suvKkN{oflOK9NHm{|roD zWTHnI4_!5b#D1if_)W98=F*E_nZ53e(0sS^3du{AF6TTQG%x$IPxkA+5xt~53G4@H?loGVeAC~zPYdOGEW6TM8I8Y$t6Yw zlO(pLou(~tvWX(w&d+bj?cDEzrzO)2gPDBL=)7&rmDT2ND$G2H;Yw zY3)g1t$8^Qf<_#wH<3-+nk>#9 zv$HIrY*Arl%v@kMphEkXn&?zisPv zH*P~M6cso}zR@t`7lTg){Kz^>Gl`r? zZ3|Y^7h6M-Rb&7gjk$)#&Lx%Yw{tk!tS}E7^qyd{TdjgI+Jn>Ee=M~VuL$>z4_i%k z;>IR?|D&LtVF}Bpy;Eq!N+P4Y7ZFOiJ;%7eQ9PKL9?(0IcK`M0)#JF@Z6gpnTg*1} zCHso@2s{7%?5ma0w&=cu5ceh+sb1o7o!ho^9tQVe9s4 z9`0SX4wkX(@r^q8c&@5ch;%mF0GIry9yBeN(oHB2^lVqrE@!T8jU)w|d3WM*m zPxq$pu&s~k{26DpRjr`dHNJ zasVCM|I}v8i0Sf+7PT$Xg1!63F82CU&z@SYxSlvHK=1RE`Zr@C%lwyjmdRqq?(|p7 z9wWd*Idy8KS@UA$lm|!Ic9Tb2zJaclU({QlF}eI<34C>YvF9L0Pc_k5R`2J*wmJpg zHFGb8#+eQ6q9(tij@{G7(_-*pEC=EE;HwJ-s0TJg_i`a(e8Z#|57x1!5fFElj-$3G z1#5ERoMmj7YyAi4=7(@|Z;L{_4}MYYqnuEpN;03pe zqX>+!(OyQiCMqw{OE|{6$c5Qel4D+To@T8cO&hZZA~DX{yW0lFF|H@w&Av*vuK`}Ir6>E`s$%qwhXT~u=e+rAUjFseWvim12l^TcH+k0I_1t38*<`WpYEB< zSV`PZDdmFLI+{CiIWD~{HC2=R&5Sw!rCV_q1GqoqE_)5$^^LRh3(NA%#l?aI&>^K7 z13yIU%=XAnC_k9YSW3F|!HbI3C7^dykaJO&|8)kV-%YUli#$ysQHtE$XJ}zY(Zk#` znmdZC`Cq@#Vvc0l-LXIJY~$Q4ipE#2q{gq}d#ye4>TL{PaGAOP9CBW)xYwWPa>7U8 z8_97y2|JrsOXNs--1iL%xOV>p?FJ`YSvexR`131M&2B`}9_uod8t_3>&v(C=lg${8 z7%{w-I?x^NHZ$3qeA#yTY6YH$O^Y>YUiVU0UtG%jZ9@$oM?P-VjVl-ceU~|Olh&2_ z&QXFf_YYG2-`b!!Oe2LHc8eBV5H3M%d^sk=1?TWk>~HdsuQYh8^~UK@AE@Hjj$(tp zj#HSDE2eXcc?wU+C$t`qCURMq1^BR0diY53G1Zh^GCMD9~O+^ zPV{4!*>)T+3o|}rNj+@6r2(vI3|Jud&o+v(ZsT09$Y%xV9HVL&Cj_RweJMDMqz6YoH=vG^MyC>IeLOqJJ03GyM#|2TRms53T@Oq z>^OENIF$%JgDPS9Niq4U3iG$5C+V;KTAQ~FWKczFhTx=V&nZa+Rl(yYPws$^%`0-| z8Pg^yJ&Ve#U$Acq&9wVSGe3R?YZFUK)9drbzdFEh0Km~#^FDL81K2xImrS$)p`qg7 z88N8dff~j}tu-i$S18>J!*R zATVRelFZTx7RJVUYuB#Ldbx3;d&VD5PVPr!0@ObAuX=dk=FR3Sua!M*iuW>te>wnH zYSyQljY%w9v3>j4urU8j>-KQ>4Hs|y^BdlglbgBnOOW~i;G*M@=og_mWKrhf^ z;voy_Jer8zyRTlosx#1Xo&Fp@Hd{82opKQ;8uPNU(&@E~?#Hn$Yo&xab20^d2I>uC zu=nZfv6StkZFs%pnw59o3|M^7oR2oBT-wxp-(8*@#9L zC^!hmWDM-ke`;T0U?WdA`a;`fW6I$mrY2sSmCRrVpLhT}Zi0ydhu@d6bE`;D35T06 zx92dnG!?~dvU=sqU#W8za7&j>5UJs4QGj-D!-j4l))vY8DM?PWLrK%)s^?^i#ApXC z%)DYJ^7PnF(=bDFr>y`n^dehyIqfjC?}L+-g&xrPj92Kn<(%)rEDImuy88YD3lx(L zan(cv)^~0ntO?iN~D7Ge!K4a=5mN8y37ZIi3mEbPnbFU}xzw z-VSdvkG?D&#p5^hSI_v7)f(@(`*kcY-g%FlsNMpC%iI8r@k33HsQ<1?B_?U~#c(Qx zuM9HTX~KRok`uQa=f-NXSKxkOb3Ea74^C$;bp~IooK52@Ngbba2&U8Qa9BjUf07c5 zY~KwaZe!60la7T&e(672c5tH9#t$Wv@JKC;pPreIXlj+lP*qHU1?~aJWJos`&7E}2 zcQ`uu0GYV?7>VMok5*0pD4K7@;Er;RFnY!a(|y8SeaBW96WcjAHg)TSJJ0n6b{Nvn zmS~OXDvd+BxdQi^sNj8VE4O{@s{GJ6N!6mX@u#@3&P0st)>*-run3m>RB^FWsi6tZ zNDVnwx{qdnoHrA+CI9dfJIf@qw@UPyzqmJ6;8e0#<2l7J^m=C7g=5k=)co&wS{_xQ-MwC!&EY4e4 zUosUrN3x7Bh}>dR(f*p7m`NeN!Z96_JReOIPmfV>y-aP~Pha)qOTEJS#}k3*=hjN) zfB$Ivy9so`ca`Gr_J(jL!$UDo&5ql@+k`LN5`4uf6;sB8yoBhEow}UH+&mBAm|zYX z^tlDET-x=5r51SvwNISUixvO4n6_!@NGjiQV@uI=16oXMnl(C?4|UtHUru_HTJ1fC zUTPR%%ecsUUS?s0o-4unV^T<%qLlwB8qf${kt=Lz7LdZte zi$8u=8*zE{Df*{((N{E8yV;me*>v<8AhX`Z0O7i;GwirNNG$C?rvA-XgyB3ibVc-y z{w79dCM#Q{gk{Q|tQ2E3Zp0219(S94YX*0jB2ADnOVPxz*D3s{Hi{t7v{sWwr%*Ki zE>DavpYrl;;)~a)-{W@kAr7jRP|kOLj*-&tK?M8O1qSBUMuQcnKV<5ZDV-n#JA38I z`V}kKAatFju3pw51-AI@kB?0x;vV3WMZWFT3LQqPq_8zy82lU;(EfYrEa5d}d~#4> zU%H9N+;7O(X1Gg(M{s)H(D#!aufQ?ctlhq}7u>vEpm4nF-|h7DFWC<5hyNLVCFXh* zJ>ME4SXVEa|EBCU#fswv+iRda6WP4LelG^m#*Uoi{|mQF7}#BNxt5-e+Gq+tvMplfzLL^wW6{uxPE;1n z2WN^*PahcbvB2xC~p185Mzx(adSF&$nTq=!^B$=PAt=+S6>%Wpq zlucqVh<`iZj5KfR0-}~t4NSJpivo@&;&KWAqf`AFj?bxh5TIeC)5VKO=MpJ-i-IwH zd>f688AK)bA0PA^v9+}H%dKJO+b=qHTp;na!(UGaSII~;% z7qVyG5>*WzV^kk#{*TC)MQD#w&Z?Zzd|s++V#_2<`m|B-{nQXl3xB6%-bN z6Um{kvz};-pPZbU=GAD>p4+_vbmT1{sIC(@t7wt39aKNND(@W!IhXQ*;lLCSBZjwG z&#NxVoN~p3&vU0*@J82EL*5v7R<*(*Px*0F8{*N%Vgqh?-DVfWgMQJSAUAi+hD2sLt4 z{<CqO-z+yOFx-KmSHGs?YESIsC=*kV$y$WgU z(8~gxRifNxOee$j{-R!B02930EFgl$#GP8~*v{j3i@lR~jj8uZx|qU!jd_EVq2#n* zEM|1Pu27Gx`!MF|{Aaz=%7+SFuWhDx;3v%MJMW}kU}4_!dU3HWhH6|_gkIXrh3~Z_ zvxf_BHCbH5B9(6sr5D0^^7NF^iDE`T5vcZw`zEg0Ev8X4$eD75%ZOw;hHh1Pqpo@s z?g~)Pk77mc4{x;dgmwsU#JIFd+{YM<;*11>bu(e&SM}G;|4|*s?%73 zz$s(kXvx8CA}L|PGS2-bI0`AaR50z`DSL{Pgdz-E@W+>8`PhU8NdFWjuf*}otI)bbKqbxfMHhi?Zlc6 zRSEc|@ZzeR*EM|kj4fZMagRvASe!q5pXT}^fbraK3vC^PR8)P1Xll0&!9sS-DlXDf zpwPaGX0G7q*u?K*O=f<6`%nlLw!gvb-U4<75SuxI%y0kCe(?Bdg7Tl3AE(;*%>eNF zp&fPU(o^u{5e)+(M#L5Vxv^0iPV(|DIG$PAg6O_urYW4U930dUIU+b)RlS?m6B}MZ zK>;xV;y{=on*k!X2*s4lL!d$o#c?sHsEQDu2yC9VY`+djNKvzEQxCqgdPJazDDQ{# z!dIh@5wT!?k5dSqg9BCv9O)AMCs;Ax-EmkGw;|HN-CYR>hf5jovo$~}qB-0(;5#56yAwAi3np`5Xm|GBHUkjJgJV;fC?fG6;uA4TtXXQFUll0iYw|Z+j)4U} zix}70%*-XVZek~II)eu&v&Mb@$S+?+(0<93gDUQl(pEvg729|1nh&0{pVCGK1_jDO zh=18ojIOYpy?S)J@nKD592^`-;NzEj0P(WFdbJ~;!`klTbS@%-G>nGY-y!nV+g>wH z&%i(s5VJ?S-E+yYkPvXHQKgaWnHthecM0Zd$&QcyAPp4XX51^sSWHN zk2CC9II)|dtNU^4YboK8(`WuXbO^|Vfu_an$6XCS@!(}mnhM2EiPchWcf^klkLiBB zSh=@zXVS$X9reGttYA~i{^i6}bkjctw7zt9HsP?c1lmF*_<~dUY97B85`JPQr;LWR z#xmdllMe~#m@FY-mEvuB_=!DeLCx!nF3RgGiZ(-hW&q7ix_`}{_ zjxa1%(<}{*%OGcr=CW$nYrm%`m#F95D1(@ZQa0yHV3^Np&7S@xnpk}5Hunb<&%4?8 z3%L-&%*`uag2Vt$M6lpEod|kHdjyRiUUe)~CQ?oUP9QoYaS*WCKJ%E_;t)C0I8#J) zU_~IySdpMi?050FjOtmUc*{QwldH4*6&K6Yg^8GZ?LMC@q%b6~C1=)(4S6?AqOZq& z*d+#%xwSY;Piq|dm-(Q0ZTX+6`s$Sm6A1r7)B#Bh<$T6wY=?(b%qazU5L zD*D)YMdsC|+^~TQJqUSiqpjh7*NTs9h#4GyuC@Jd)>TK8vZhyZmGxr1V5_FTYkNNm z$5ddao_UJeYg99qYNM~a9yj=KMU%-CQml|tua+Rx=1Nb$|7F`tJU1$I(TIK|y_d1RdfMgp zk4VkU#H8p^bE@0sDFQ2l8#-)Ll%+r}y>Uh>g9_vNAAGM>gokM8@@fQa$Ws9JJwtq_r-9xHLIl&L(Ooj%|UcS5|E-o${C`1+f#5Ud_c5UL94YUJQc+C*_pIx#)G@2V5*OlLfSt1#e z`_sT4{F{$q zAOgbVp@B>1&e_|5Nw^95IVIn|?RuwXNKJJD>+cJ7*Y|xj+2!!z>Hc#l5yY45-)4+s!+}odGGLG$Nk2^A3KAtEZKcclVm6c5rHdm z0!?}ls|8|F${yLRaV~D%T}!M+aA5TT1k&9|JyYtrv!{Nk&RhJi``)kbOMX7Tk6y0M z@-BJ&`02A{|!~K8sCU**iDu@In4&K zIWoUN)xG7K8|bf94V2_WmH@&p$h{LMh)d;6Q$NP$mMAAaA6Mw5nVp z7q_;y)_ZXD;ETQAa@OwokmI|MAyAw10EFKU9q#`D&UnnXgjzQeA|w1C4JC)K<+pMC za)<(PST+{{?u+5(m(A<>T25T!kmrOzz&N~JQ&WV86Jju=;(^}ougSyx z84^tc4|tjWdByo?HF&a*fYg1GwDkCV_#F`MA(DR5)VPNHRpiZifzx?#Kw@XGu=E!V z*`&<>J9WOjy>SwnB_~c04|@v~BaxBH*uh`IR8U4L02DS_O5-O3WyL^W!t7)3cXS&V z|3&gChY|8@K)&11JetJ_FI?2>qe3zP5L|(p2T3b} zx%^hb?8{YmbY*pi`y3E*ZSc*-Yr*w8d+E|@xCfXlynK8YGmWOBhu4edGf2+lHdAynVrBscZnn(5!QrDIwl|TsW0GV96ND>9kc1?2CSOd>qGNNWJ)o?0Rlk?S|YhK zsAF@6drd{^HXdTZhQmYD{Y#$?7S2;syH{BiaE;CYlyW;)V)iq9v0E|h6q@> z1nWm4Uh=`8Kt2t9Egf&(stgZ!*9LnudNMad0IRQCVL0tZO777*0V8ZA3eYT(58>nI zA1sKrfQ4g!8N8eH>L`Zu(!lWH;9vwymSa;~Yh<)=BHQ&-faNB#T#%O)5E1!VwL?)= z^*mNg1YY#L-E)O@Omq^B*oL;yk^h}Iz>W)hAu7&P>XXqzIOzMf7Yy7eQhV3x!J*P$$my1 zSYS}z`@~f}W$^ozk$xf%47SW?MGYehx4(C0kJWvbiwlLy?_CelMXJwHc%ywa2i{y; z1r100JC-d_t#r?*$`I$Z!(I->w9;&>6nGaCjut)wI4OY!Q6Ey3EMdB6o5KOg6=+_l z*W`XmcYH2Jfpt<{q8uh+Xc0q$;zQnnWzb&ve;)z3X7@L~eHat~O|>FgZqW3qxaT~X z>`OmHO|pO`h`RspmoJ+jwc`MYLOMMp4PEW`>9l&yk59e8xRq>ktYq#=F~A(`|N*rCoo4XokhfNy7@L{IZy4+Ml=?~gnocvb|DtL<%50d zu%!-A6vpOqd93-t7SdCMwh;XKNCbGgKmE1rJJc4>lV%+v59P;yPSEjI2i!6kG8q#k z=nh#glJ@OK9u;Z2LHATHI;eNrNMF%1mzev*AwB)d*s{!1tp7&0(hGYxz1TK9)K3iX zoSPEM0{c9HkPCp9Nuv(3Ii#OUszgrO7VF^dawu8`fuVVu;-E5Tgd-t;2vZCN+q;}3 z(cEj-U>4E41^)+ytAMkc8i!mhN z7}9jW&T`-0y&b26PYELuB!uc1f(X2z%9nxe#L*7j`~6FQRAQ3p#n)bDC^kW5AtZbN zy@*%Ox?K2AiUPZ}Bw53BOph1rqK4w>w3o9>-8Jc(RqgK1wGH;>_kN5)oOxq{KHzuEz?q*&m3*89NE; ziu7YE;qI+{(B!$qr^|0*yf69TtTuR5J8($m^Y#5{oG*9dA;R*g$-YQh0(SY#zMd~na0|unn}HJ}mhczNq;$fA&N@!p(Bc&Vo` zr2rVDaD4b%N&za+C$UYDW;$juNm-tj)aZ@;ihZS-WX$1Wz|6MW!4aU_@2bu`3%QeN zg~~K9`jBrKisz#!p#c)2IERj!jz}ZqjxsB4Tqnlc!oi<4-OKK74orsy)ASZ-~~F z#%>Eerqw8kU`S{a!_C#rh1^UXDVd_Mg7k}lAy{XwT=|y4JXSNLr2`)-Y_VH-M=x?e zmEq&;AL?2I{mmCppA}cOW1!i}9|MON7Vs@nR*M6XfIGAkVKDYvs%%1m4_xsltv&<8 zekJZ{ zws43AUnE#W_9b>I6lQG0MvOs2PHRh!5ki#&^gBi%BI1NJPWegz30s^32eoyS-b>;f`tkgknLE(0u@Qu+z-fM+_&Zyw))U+^8f8p^wR z$|pmo4KBkoY^ijdO8^V?Og}nm^By~>s;EG_bmQ?VNp+XQ&f(j4>^Oh+>~aiM@V361 zImp(~ACyJ_@Jp1>3B#0q^l zNMM(*7^OFFpxpyy`quJcoRg3(A&d&@Zq2VrX{4XUK*X0Y9qHU7hGNl>I8Uy@h&!-w zW61DP-(g_h;W!2Ulph4Oped-K;}SlbBoP5R@?Gw$1P%CHyLL?+`3iLcKCqfs(dhs~ zXU?4~fIG6@#AFg&v(Bn>e0Awn?w@Go;tXgT!Wo!|wbi*?#Zf+4ie}Ryg`TsT$HyzQ zeUz4(+BUwy6HpY`j#Td?b4A1#pipuiJ_>HX0^5Xuuy7p2$bkZ+b80pqVg?Vq9)sEh zc#!baVvGJ4$wD>5v^dEbaweCc9d#a+jik~&QB-sl&Y8%LQ!l`$A$533_soMNA_jkM zH$(?jo!`!a-Y1LmuJto;FIQt3>jWoy%04Z|&}+gWgV68eo=>$S!y4F`8J#W9378`D zTBuN&j_fjO_D0m&pU=(Jp=|-P6&AG)4#o_m^*|*o1X=7Kee;lqu?W!}xMX@*m%vCA zw}+|XITIOKLIxLLOB*u6Mu4ew7?_Ft4ZM8HQwuBoFH~p}e?OO%>Q^_3TXs(;>1X$? y<{mZ1M!lJk3Ihts9_|tOzf1T0j%B_W;hE5Hb!3k~_I?Kc{jq9;e##2FL;nZ=mN#es literal 0 HcmV?d00001 diff --git a/codes/QLearning/results/20210312-165244/rewards_train.npy b/codes/QLearning/results/20210312-165244/rewards_train.npy new file mode 100644 index 0000000000000000000000000000000000000000..91008fd9cb9d539462f0955e80c2166c71ca19b9 GIT binary patch literal 1728 zcmbW0xl3GO6i2lZ{41t=2~06q1+ht!PO(Wbl??HTf@H!Q5kb;eSP0r!NRl?e!a^a4 zib1l7MokorK}8d{sBwvFVV7~#bD9S>PILI(@9y_ach%F%>U42Yb5Uzh8|I!xL3tvm zXm}JP6G2@zYKk(?YqC*ocx``@Y0QNMd#*n7A}r_!WvNv1VKVVDaqnMg`Z9m>+xdUX zUKriV{2sa!{TS}U9^ONA8*G8QytkuY!x}gVJ75W%fN!9AU!lLj0oVm+U_b1JLvRef zgWsWfM()rbun)pMxCuYOUT7ZYc#C}reuc}>emBtX;WS)@U*Ifso^kY#n4j~->yF~L z2ED^D+PTap-LHMk;lBXg(=W9B?xBtQgPw=>VVw1Sw~qFY#F@u^{>5&;yD{y)*Rg+w z+t5CLqVLLMUHkPOYTu{GV_%Et6KGxOz070(>Mi2p`So)@|M2rJztR40olm>@*75T@ zJVzfw=d@qzI*;?3@0d9C8QM9lqjt_6{H(Krb}kvW8|NL(EA>~~zxO$%j`0U*`}aHb zU4QqgpL;zb-nqT+1$N)<+y2~JJg@h)Z{ziMFUDzi&)!q*9-PCuytjSouQvY`H32Zb literal 0 HcmV?d00001 diff --git a/codes/QLearning/saved_model/20210311-192256/Q_table.pkl b/codes/QLearning/saved_model/20210311-192256/Q_table.pkl deleted file mode 100644 index 159318d74d772da055d704eb5a43d57796d05537..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5039 zcmZ{o30PG}mVh6t@)Q&h_g!Of19vf^CZeKVMJ=O2T%PYfo`OD>bGgJujd?C0ilR}8 zK?t&#h+6=~2q@5fPS4Uk(@WBO&vei9zRbS#GSllh_uZ?pXZrGS-l?in=db15s=BpV zi->f&1`c%npJ%8m!R0l4dHJvVQ**thH>LPsV9m-nSI%=!Fx=-W3IzN?#GUKk z=PeDGx%ocR4K819ejt$I5$X2$48Pa(d-jzUTDEQoOaA}LeMPx`H-sl^F6%EsOMK31 zzu)wDP1A5g#B=%i1;s^%=_x46Ee&w}$m{{!A&)rol|WQEL@OAO9nE?ELdk@I6%eDq zogK?rrVAUxu+oK1VOw1=$TACf&HTc-Ft`F@6~twg!;tK7HZ3fMq1jQa9QGSUB@kZ$ z3DdJzb2@cfiQgzmJ#Zj(mA~YmSyY^A`11?*6&XH%fxpn49q@aN!u-Phv&~Yo$jJ8w zN>aCMu=7&A``IL=xC|1rWg^l8-U2<>yBvmTjH; zgo2UN6^zP`kw$vFq3M9p8vOe}{IK-^Y<+8l^rHW>mmT1+#osZFga=RmlyrHCH*Lu#QWT`7sAYSwl{&fLfOi?bMR%!*byzBm zNmsDUgyp(6GH+I%)!G$#+`5+)ykfU*WnP*P<}-nAP$dS(PsjeT6vWOFe4B339eltk z4mmtukU_PqQ#oE5IWr%-SceRutWvPrgf+T0*r8+!WvzmBHf4Q?GLI?yu>vpVucgK@ z6dXm@r%+28)v`{HsR6s`KE55njwfzfO+DP8_qhZ#foxE)(S$5r8|6SY31qW^EjC1b zqAPuPn{MGdc&C&dlzj?s?Z7LBP`ZloYBY~m@D&PX;``XR2|vVUp==e(s|vQ6uwB#$)KU0tw< z^I@kjb}86x!X90Va~K|B>{Z~k8Tzx~+Ct5EYko9!jiTUetOV>#r+U_3q!W0Ju5F`k zPS8DixP;oNHCiA(f#fRio3KyU!X3zdf#fO3w;=~YkQDC7-nckwK0+1Zs)?t%W5!Gr zzqt`qV{XMBRtLu?VeK~TV-*exAfTYYghE}5Z~#RDC{_SAzz6{(cn{}g(8;5CaU=t- zy*Z0c(+#ZQ#H-XDm`rVWTMF(?hGTxb%PN!z!BkLc!t1&g=@1SH;Zq9AY{KDZU6@Nv zR5b!GZyG_tfSyg|)UcJ+G&+Yp8`HB`9fX(Z-96agWgU(P;HZK(OgN@%Lmj}I0thNN zZUat)0CTttI+o+f?euUhS3gF*D_+G9ma>Y+@d7=f7OGs%si_CChPn<&BjBVUPAMoi zp+eWjI*8MPs8n#qMx1>%O4y3`=_Yk7p}WH=c!;XYVj`)M)w~3JG(7siY96HKD7=of zlUzcm62dtJ=S`^AwJ{Fif)Fk$c*`bS3K14?CvvcZUyIY3*vg}f_jI~jNEb0Equ-8a zsqxL3F;ltVKHi}!s@_9aWHK%b=8A$E6W-RfSO;@eFz+a+wK3P8btHo-R=9@KBaX&H z2~GT@xB~EzBWY7eb?gCY#V4A}r*N z&?D@n`bqQ{uZyo5YRKSq>7JE}VVNCfcC9y~#(u#popv*^@F>cu`8ywljfc-jltEyHz3 zAa@nqGvNbWi*g`s0%=#!VMFdegN(r&99$gVA^)-%(q(EcUP+zU znZWIScky^^&cqhh6BDLzIGvz!*6B=sGF=>vJ*)!>!i2xnwSf-fuY~c}3O;8u zJ|AMp4)yX3JTs2!C^(c|y@S;lT!n3Pic>D?z*CE{DOA*UXdN%@-Gkzp@;5^HTLoV* z;fuN!<50dNlrJmz$fkV!tS8tUOI>k$@U2+!){76fM`4il65gIpC1DR<<%G6n8lB{q z^)c)46=8f;!PiXqx~?TTjBg0zn+m>VGrk>SNL*njk1}tsZ950aFZLEuFICe;H>YkZ zY#EFVRL_aSM>iZgN+;L|{!So&ui!f-d{@_oIgo!4$UiFho(=ha2qH@sJ{+-n1=Z5^ zA?)?|+(kczyWW#5;O2_F+XdM+bJ#z~DCQ8pyg4Asz--7?GCiD3+!X%~IDU z+Gt~dm@s{yWfH^Ern{xUI^{`FEXSM2c3^GNU<$6m^LVu|0Z&Q{w#=K9L&sBj`RA~_ zFqE$273yUpV~|v5^kBPLKi%Z3TS;y;qb>XQ?Ba zyVsgYO^1i!2RxdxI^gZO)Ho-df|2wgURsln!2t|U6bB3+W_eCxX&S>>>e_GzK0-_w zKGHH7#nOh4mV&4KE~KX2bfRRupxdZh&7?YA&^M`Ki|mQJI5Co1OYl9c2(WI9k?M>d zYxy0=(lo}i)V0YDdV-iRdZJ}AiKUI6ECm)?PCt{VgYF%}3s^yY*uQt5JffwC=gg){ znKI;;u;oQQ2Q_aEXYv%O&E%<;+ZR}x#x$0?Ho_slC?-r!woIn8w8=B1z#_{wnZdzz zITugUdEuoUrE09s!|T+K{V_bhE&L^ML6@Rpyb+;I;0R~km?=J(Kg)7Ho26-_u++6F z4nI{)m_Nrdnak4V&yxa+FR#|ob&Hd*Z7=WE{jeS{$J3L=^Z@(m3H3kyYs3CmJ*m@q!gGFigX#=j&5Pv;yv-(1Rvv$_mEWbEN} z>H)T6#YRr7r{i>C*Z6&`@^?pX8un%EWZhUQ)ft^``CZ1+G?uf}wRi`;LQEL_vSsoL zOB=mX3M}*@?m36l72>y=spomaW~B;S%J2l_W8yVt+ z8?egqzM7?JtYN8ZQon>H3<7y zow)qS>?89y1B*cowzJTE)HL)%qIMdzMiHFJX)LtkyS^{uUf9R zu{4eCEOl*yLw`+7m@Z$+%Gklursqh3MYooi$Goq#rBVM&y!TV5pJ)Gu*UNj`{>?|R zeH>NJmjQo>jU1B4sg4Vbo#KGuyDZPUS(?TkmU?N*W^KSg)!uq;VS>vg|C|5pCDagC z+|x^_*ZG2lFPYY?bbsOwYQ9^Xeblgjn_SPx|7I;rexBZGeG)WO3Z8rZ^oHwCYk$)7 zkmqWabHkn5cGEU%(b2PPqwx_EEuqJ153A1a5j+25b|DYflj|zi#zc`W+2=SfO0KK8 LPPj|#L$-ee2TxG% diff --git a/codes/QLearning/saved_model/20210312-165244/Qleaning_model.pkl b/codes/QLearning/saved_model/20210312-165244/Qleaning_model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c70d88ffb0d1c3de70aceedd5fd1f0b67995067a GIT binary patch literal 5039 zcmZ{o2V56PmdF1H68<78h$5yjjbRi-)G>~VileOzmV+`nBlGV+e+2mxUQ@VYL`RY+ ziU}OUm;fCkD#^s&dzaJZaJPH6dpYc0*u5NQ-|K(3GtS*MA0Dr|Uj4qWtE#Jt&Njj$ zJf7%i&;PlGdg47k%U@WuE0CJ+)4r7Q;$UV*oM-DRSM>nDzbqIG_;pcPX@xgGP~fW! z>ii`K^FY2l5IkOYK-MgiZVZ>iuQ;0WS=gW_awMa3kTBs2u^F z=hNEqLij89i%QDNES*6MAt%$0&nhM zzNI~QB@F95cr|R32L{<;L7y%vO@r847_1;Js|JST4&czzau}LBkexjNtE>Ws)k6IA z+|^u8&8Y}j6{$OSrmhN96zj6`R4Y(aT2N;B10{h{Ju4XSS*1m#J7(!hU1k;ef)%M7 z*E{!8eLFZLrMwCfa%CWv1broDzV9s5A%kVU{5)M%&JUPqXJJ<9Kn0AbfsqQ5 zrYjhg8zV{P`9i}1$u%%~x`r_d#`<64Qf!6udsom|PfM7?C?hn3W~vRVn^5E0B^`!ZV=AXIZ|gyvh=v zE`+HyFipYq+*l6GvjTpeDcY*^YnV|BZz!1Qw_AHl^X*1nm}PisAf?co8^#SxNG;@H zn5`hK4(2FGpJs$fQgfYF`JoP(XOB<80+`=WUk7h0STM~9w*{8K!mO;Ub)WHN8^fbv zk=elVDJ(yiuH%IR*{_YEYP!8HkwHgW-*wqj#6oyL<~NWxk=iO=xNegSP0(EAEL z(6HS!;#^RkfId{)m#B=Q92`_zw&13Nq7gl2D z+k02zJ$At_kbDIJ4F#q#z=iA(NTGrv2eLB+kr5w_t3)5$&DemK?UVDA7t#Qs+7 z&7^+nILQ8TJ!j$;yefoZAp{kaXec#}Fqcp!gmMMo5UdbECM|Z9?wy0@7jq|2>-d@Z z^dw8%u4rn*wn^FIU!(5*tFV2I^izcZw1P?vyG&z<3;2ovb}OiI0DD4!H0h_-)pT=w zBF`$SE?S>~PZnS!yHp*H$LTqq$5UL2A3YdvQVU*VANC4kpMw1w4wy!`%Qz^EY6XWJ z#^De{=2aTrqmSuQc?maNI<6=ppsu^$M)vLOMOBN7SM5h9Eu?#1REG8fr}= z!bQ{xqFzCRgE$&O$eU5J8k^S7UNeHKCt<^2e%($R>`%rU{0v9&Suq!~=Zqbbk;M)) ziu^GJ$2ByW#z0s8gvg&%(Co;c3dzrvUb>LE5_`B?o(|^;beE1yqMl_DiR@pdn(Y&) zH&*;sYR6Bgj$ff?90sQab4I~g4d+ZF(#5n0rd7dt2lMgggOP(r=ma$l!ZQghcN9-& zR#813!$wGd;LJjzY0 ziu%F|h0w5j;%w?%z^;){i zja_&ogvScHG<2KBaF@^{gkA-G4&lk?qmjnTR3qKNqcKCISFWvBbekIDsCs@H4|2yl zbd!BNoKD>MXZSU5i2 zmtG(pb#_WJ-HIwh9Q|*G^h1km3^#v4SMfnSRnNn^@S(iG-(f$A-@7~W24NR(8fEx^PL0Ryx^Hr{+%yi~o59;b$5{VaA%@e-cH&gA9nQyZOJLha+Il?(6_Vf<9V-)Z=J z(}-~y{~(NiRPZy0@$(Qv7D9YJd7Ez$9^+oXOKEW%)>pAdj~C3q=FMBh?V5q-CU3%C z>85`Y`F~dMFB<;UG`z0-zlr?6EBFsb{y#r|gxS=*XjUS%Mzc~qUEA~~UJSAi)R5w( z$hEbOU+!m74z&#(yjL9}WL&8Y5iBXUwoXeC$(J7~2rZ8o+J{W^aBc=?I-o z$A0>l$02NN6!pD3lD#)KB+=9O1ojruAw8dm>oWURxHOKfwIXcQfo!!E$<{O`I#er4 zLKqxvhs3aTz+S0%d6FxR@L>xcr4A22<}s{3fW3hNdV&4i!F*kt>B^a#)LN?gx}2iZgL1l=V*FlEqX zDLoCxT0GB7>T}s&MlXrxW6ec&tppL_1cuw{iEOnsf~{$cbQ2gUA)G*x9WsiolR&am zyi5R_6+OTcg>*xnjFYLyGjtOhL5daI@W=`}o+Ccpr^7pTe#mc}^uTBl;RMFm>SNhz zYaCnCnB*ofUP3s5*X)o9Y@GxqN`;+(Y_(IUg?V?2v4->Lr!&bDqEhGubz=YPF%0Xs zf9_-dhGp24N1fXx%9s#@p*awKFdW5Z1B;X@xb@xb4qyH)r=O7|v>EK3<7EBlpoYxc*J97?T~(wxz&Y~9st zwUx=%G{(BI^Ax+qr=z=k)ac*c27DL!JtdUl?NiJM%>F)BNi+M_6$b%=fE4X zu5OdoOn%>%{eZ2uwzD-WQ?iYy=(f10TNC3w9{J7wr5~Awcv4<|WZK0aQ1~NKX2y&! zePep%gVUD^>0W_-