bacow/easy-rl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hot update PG

Browse Source
This commit is contained in:
johnjim0816
2022-08-25 21:00:53 +08:00
parent 4f4658503e
commit 80f20c73be
34 changed files with 1391 additions and 1695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
projects/codes/QLearning/main.py
View File

@@ -5,7 +5,7 @@ Author: John
Email: johnjim0816@gmail.com
Date: 2020-09-11 23:03:00
LastEditor: John
LastEditTime: 2022-08-24 11:27:01
LastEditTime: 2022-08-25 14:59:15
Discription:
Environment:
'''
@@ -18,136 +18,102 @@ sys.path.append(parent_path) # add path to system path
import gym
import datetime
import argparse
from envs.gridworld_env import CliffWalkingWapper,FrozenLakeWapper
from envs.gridworld_env import FrozenLakeWapper
from envs.wrappers import CliffWalkingWapper
from envs.register import register_env
from qlearning import QLearning
from common.utils import plot_rewards,save_args,all_seed
from common.utils import save_results,make_dir
def get_args():
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
parser = argparse.ArgumentParser(description="hyperparameters")
parser.add_argument('--algo_name',default='Q-learning',type=str,help="name of algorithm")
parser.add_argument('--env_name',default='CliffWalking-v0',type=str,help="name of environment")
parser.add_argument('--train_eps',default=400,type=int,help="episodes of training")
parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing")
parser.add_argument('--gamma',default=0.90,type=float,help="discounted factor")
parser.add_argument('--epsilon_start',default=0.95,type=float,help="initial value of epsilon")
parser.add_argument('--epsilon_end',default=0.01,type=float,help="final value of epsilon")
parser.add_argument('--epsilon_decay',default=300,type=int,help="decay rate of epsilon")
parser.add_argument('--lr',default=0.1,type=float,help="learning rate")
parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda")
parser.add_argument('--seed',default=10,type=int,help="seed")
parser.add_argument('--show_fig',default=False,type=bool,help="if show figure or not")
parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not")
args = parser.parse_args()
default_args = {'result_path':f"{curr_path}/outputs/{args.env_name}/{curr_time}/results/",
'model_path':f"{curr_path}/outputs/{args.env_name}/{curr_time}/models/",
}
args = {**vars(args),**default_args} # type(dict)
return args
def env_agent_config(cfg):
''' create env and agent
'''
if cfg['env_name'] == 'CliffWalking-v0':
env = gym.make(cfg['env_name'])
env = CliffWalkingWapper(env)
if cfg['env_name'] == 'FrozenLake-v1':
env = gym.make(cfg['env_name'],is_slippery=False)
if cfg['seed'] !=0: # set random seed
all_seed(env,seed=cfg["seed"])
n_states = env.observation_space.n # state dimension
n_actions = env.action_space.n # action dimension
print(f"n_states: {n_states}, n_actions: {n_actions}")
cfg.update({"n_states":n_states,"n_actions":n_actions}) # update to cfg paramters
agent = QLearning(cfg)
return env,agent
def main(cfg,env,agent,tag = 'train'):
print(f"Start {tag}ing!")
print(f"Env: {cfg['env_name']}, Algorithm: {cfg['algo_name']}, Device: {cfg['device']}")
rewards = [] # 记录奖励
for i_ep in range(cfg.train_eps):
ep_reward = 0 # 记录每个回合的奖励
state = env.reset() # 重置环境,即开始新的回合
while True:
if tag == 'train':action = agent.sample_action(state) # 根据算法采样一个动作
else: agent.predict_action(state)
next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互
if tag == 'train':agent.update(state, action, reward, next_state, done) # Q学习算法更新
state = next_state # 更新状态
ep_reward += reward
if done:
break
rewards.append(ep_reward)
print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.1f}，Epsilon：{agent.epsilon}")
print(f"Finish {tag}ing!")
return {"rewards":rewards}
def train(cfg,env,agent):
print("Start training!")
print(f"Env: {cfg['env_name']}, Algorithm: {cfg['algo_name']}, Device: {cfg['device']}")
rewards = [] # record rewards for all episodes
steps = [] # record steps for all episodes
for i_ep in range(cfg['train_eps']):
ep_reward = 0 # reward per episode
ep_step = 0 # step per episode
state = env.reset() # reset and obtain initial state
while True:
action = agent.sample_action(state) # sample action
next_state, reward, done, _ = env.step(action) # update env and return transitions
agent.update(state, action, reward, next_state, done) # update agent
state = next_state # update state
ep_reward += reward
ep_step += 1
if done:
break
rewards.append(ep_reward)
steps.append(ep_step)
if (i_ep+1)%10==0:
print(f'Episode: {i_ep+1}/{cfg["train_eps"]}, Reward: {ep_reward:.2f}, Steps:{ep_step}, Epislon: {agent.epsilon:.3f}')
print("Finish training!")
return {'episodes':range(len(rewards)),'rewards':rewards,'steps':steps}
def test(cfg,env,agent):
print("Start testing!")
print(f"Env: {cfg['env_name']}, Algorithm: {cfg['algo_name']}, Device: {cfg['device']}")
rewards = [] # record rewards for all episodes
steps = [] # record steps for all episodes
for i_ep in range(cfg['test_eps']):
ep_reward = 0 # reward per episode
ep_step = 0
state = env.reset() # reset and obtain initial state
while True:
action = agent.predict_action(state) # predict action
next_state, reward, done, _ = env.step(action)
state = next_state
ep_reward += reward
ep_step += 1
if done:
break
rewards.append(ep_reward)
steps.append(ep_step)
print(f"Episode: {i_ep+1}/{cfg['test_eps']}, Steps:{ep_step}, Reward: {ep_reward:.2f}")
print("Finish testing!")
return {'episodes':range(len(rewards)),'rewards':rewards,'steps':steps}
from common.utils import all_seed
from common.launcher import Launcher
class Main(Launcher):
def get_args(self):
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
parser = argparse.ArgumentParser(description="hyperparameters")
parser.add_argument('--algo_name',default='Q-learning',type=str,help="name of algorithm")
parser.add_argument('--env_name',default='CliffWalking-v0',type=str,help="name of environment")
parser.add_argument('--train_eps',default=400,type=int,help="episodes of training")
parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing")
parser.add_argument('--gamma',default=0.90,type=float,help="discounted factor")
parser.add_argument('--epsilon_start',default=0.95,type=float,help="initial value of epsilon")
parser.add_argument('--epsilon_end',default=0.01,type=float,help="final value of epsilon")
parser.add_argument('--epsilon_decay',default=300,type=int,help="decay rate of epsilon")
parser.add_argument('--lr',default=0.1,type=float,help="learning rate")
parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda")
parser.add_argument('--seed',default=10,type=int,help="seed")
parser.add_argument('--show_fig',default=False,type=bool,help="if show figure or not")
parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not")
args = parser.parse_args()
default_args = {'result_path':f"{curr_path}/outputs/{args.env_name}/{curr_time}/results/",
'model_path':f"{curr_path}/outputs/{args.env_name}/{curr_time}/models/",
}
args = {**vars(args),**default_args} # type(dict)
return args
def env_agent_config(self,cfg):
''' create env and agent
'''
register_env(cfg['env_name'])
env = gym.make(cfg['env_name'])
if cfg['env_name'] == 'CliffWalking-v0':
env = CliffWalkingWapper(env)
if cfg['seed'] !=0: # set random seed
all_seed(env,seed=cfg["seed"])
n_states = env.observation_space.n # state dimension
n_actions = env.action_space.n # action dimension
print(f"n_states: {n_states}, n_actions: {n_actions}")
cfg.update({"n_states":n_states,"n_actions":n_actions}) # update to cfg paramters
agent = QLearning(cfg)
return env,agent
def train(self,cfg,env,agent):
print("Start training!")
print(f"Env: {cfg['env_name']}, Algorithm: {cfg['algo_name']}, Device: {cfg['device']}")
rewards = [] # record rewards for all episodes
steps = [] # record steps for all episodes
for i_ep in range(cfg['train_eps']):
ep_reward = 0 # reward per episode
ep_step = 0 # step per episode
state = env.reset() # reset and obtain initial state
while True:
action = agent.sample_action(state) # sample action
next_state, reward, done, _ = env.step(action) # update env and return transitions
agent.update(state, action, reward, next_state, done) # update agent
state = next_state # update state
ep_reward += reward
ep_step += 1
if done:
break
rewards.append(ep_reward)
steps.append(ep_step)
if (i_ep+1)%10==0:
print(f'Episode: {i_ep+1}/{cfg["train_eps"]}, Reward: {ep_reward:.2f}, Steps:{ep_step}, Epislon: {agent.epsilon:.3f}')
print("Finish training!")
return {'episodes':range(len(rewards)),'rewards':rewards,'steps':steps}
def test(self,cfg,env,agent):
print("Start testing!")
print(f"Env: {cfg['env_name']}, Algorithm: {cfg['algo_name']}, Device: {cfg['device']}")
rewards = [] # record rewards for all episodes
steps = [] # record steps for all episodes
for i_ep in range(cfg['test_eps']):
ep_reward = 0 # reward per episode
ep_step = 0
state = env.reset() # reset and obtain initial state
while True:
action = agent.predict_action(state) # predict action
next_state, reward, done, _ = env.step(action)
state = next_state
ep_reward += reward
ep_step += 1
if done:
break
rewards.append(ep_reward)
steps.append(ep_step)
print(f"Episode: {i_ep+1}/{cfg['test_eps']}, Steps:{ep_step}, Reward: {ep_reward:.2f}")
print("Finish testing!")
return {'episodes':range(len(rewards)),'rewards':rewards,'steps':steps}
if __name__ == "__main__":
cfg = get_args()
# training
env, agent = env_agent_config(cfg)
res_dic = train(cfg, env, agent)
save_args(cfg,path = cfg['result_path']) # save parameters
agent.save_model(path = cfg['model_path']) # save models
save_results(res_dic, tag = 'train', path = cfg['result_path']) # save results
plot_rewards(res_dic['rewards'], cfg, path = cfg['result_path'],tag = "train") # plot results
# testing
env, agent = env_agent_config(cfg) # create new env for testing, sometimes can ignore this step
agent.load_model(path = cfg['model_path']) # load model
res_dic = test(cfg, env, agent)
save_results(res_dic, tag='test',
path = cfg['result_path'])
plot_rewards(res_dic['rewards'], cfg, path = cfg['result_path'],tag = "test")
main = Main()
main.run()

BIN
projects/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/models/Qleaning_model.pkl
View File

Binary file not shown.

BIN
projects/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/results/testing_curve.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 22 KiB

BIN
projects/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/results/training_curve.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 53 KiB

801
projects/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/results/training_results.csv
View File

@@ -1,801 +0,0 @@
episodes,rewards,steps
0,0.0,20
1,0.0,14
2,0.0,13
3,0.0,9
4,0.0,10
5,0.0,6
6,0.0,11
7,0.0,6
8,0.0,3
9,0.0,9
10,0.0,11
11,0.0,22
12,0.0,5
13,0.0,16
14,0.0,4
15,0.0,9
16,0.0,18
17,0.0,2
18,0.0,4
19,0.0,8
20,0.0,7
21,0.0,4
22,0.0,22
23,0.0,15
24,0.0,5
25,0.0,16
26,0.0,7
27,0.0,19
28,0.0,22
29,0.0,16
30,0.0,11
31,0.0,22
32,0.0,28
33,0.0,23
34,0.0,4
35,0.0,11
36,0.0,8
37,0.0,15
38,0.0,5
39,0.0,7
40,0.0,9
41,0.0,4
42,0.0,3
43,0.0,6
44,0.0,41
45,0.0,9
46,0.0,23
47,0.0,3
48,1.0,38
49,0.0,29
50,0.0,17
51,0.0,4
52,0.0,2
53,0.0,25
54,0.0,6
55,0.0,2
56,0.0,30
57,0.0,6
58,0.0,7
59,0.0,11
60,0.0,9
61,0.0,8
62,0.0,23
63,0.0,10
64,0.0,3
65,0.0,5
66,0.0,7
67,0.0,18
68,0.0,8
69,0.0,26
70,0.0,6
71,0.0,14
72,0.0,4
73,0.0,25
74,0.0,21
75,0.0,13
76,0.0,4
77,0.0,29
78,0.0,21
79,0.0,6
80,0.0,6
81,0.0,11
82,0.0,21
83,0.0,9
84,0.0,9
85,0.0,7
86,0.0,48
87,0.0,23
88,0.0,100
89,0.0,60
90,0.0,7
91,0.0,10
92,0.0,24
93,0.0,4
94,0.0,7
95,0.0,17
96,0.0,87
97,0.0,28
98,0.0,7
99,0.0,5
100,0.0,12
101,0.0,14
102,0.0,6
103,0.0,13
104,0.0,93
105,0.0,4
106,0.0,50
107,0.0,8
108,0.0,12
109,0.0,43
110,0.0,30
111,0.0,15
112,0.0,19
113,0.0,100
114,0.0,82
115,0.0,40
116,0.0,88
117,0.0,19
118,0.0,30
119,0.0,27
120,0.0,5
121,0.0,87
122,0.0,9
123,0.0,64
124,0.0,27
125,0.0,68
126,0.0,81
127,0.0,86
128,0.0,100
129,0.0,100
130,0.0,27
131,0.0,41
132,0.0,70
133,0.0,27
134,0.0,6
135,0.0,18
136,0.0,38
137,0.0,26
138,0.0,36
139,0.0,3
140,0.0,61
141,0.0,100
142,0.0,4
143,0.0,39
144,0.0,18
145,0.0,33
146,0.0,29
147,0.0,49
148,0.0,88
149,0.0,22
150,0.0,65
151,0.0,36
152,0.0,30
153,0.0,58
154,0.0,43
155,0.0,53
156,0.0,43
157,0.0,13
158,0.0,8
159,0.0,39
160,0.0,29
161,0.0,26
162,0.0,60
163,0.0,100
164,0.0,31
165,0.0,22
166,0.0,100
167,0.0,46
168,0.0,23
169,0.0,54
170,0.0,8
171,0.0,58
172,0.0,3
173,0.0,47
174,0.0,16
175,0.0,21
176,0.0,44
177,0.0,29
178,0.0,100
179,0.0,100
180,0.0,62
181,0.0,83
182,0.0,26
183,0.0,24
184,0.0,10
185,0.0,12
186,0.0,40
187,0.0,25
188,0.0,18
189,0.0,60
190,0.0,100
191,0.0,100
192,0.0,24
193,0.0,56
194,0.0,71
195,0.0,19
196,0.0,100
197,0.0,44
198,0.0,41
199,0.0,41
200,0.0,60
201,0.0,31
202,0.0,34
203,0.0,35
204,0.0,59
205,0.0,51
206,0.0,100
207,0.0,100
208,0.0,100
209,0.0,100
210,0.0,37
211,0.0,68
212,0.0,40
213,0.0,17
214,0.0,79
215,0.0,100
216,0.0,26
217,0.0,61
218,0.0,25
219,0.0,18
220,0.0,27
221,0.0,13
222,0.0,100
223,0.0,87
224,0.0,100
225,0.0,92
226,0.0,100
227,0.0,8
228,0.0,100
229,0.0,64
230,0.0,17
231,0.0,82
232,0.0,100
233,0.0,94
234,0.0,7
235,0.0,36
236,0.0,100
237,0.0,56
238,0.0,17
239,0.0,100
240,0.0,83
241,0.0,100
242,0.0,100
243,0.0,43
244,0.0,87
245,0.0,42
246,0.0,80
247,0.0,54
248,0.0,82
249,0.0,97
250,0.0,65
251,0.0,83
252,0.0,100
253,0.0,59
254,0.0,100
255,0.0,78
256,0.0,100
257,0.0,100
258,0.0,43
259,0.0,80
260,0.0,100
261,0.0,70
262,0.0,94
263,0.0,100
264,0.0,100
265,0.0,37
266,0.0,11
267,0.0,31
268,0.0,100
269,0.0,34
270,0.0,32
271,0.0,58
272,0.0,38
273,0.0,28
274,0.0,100
275,0.0,59
276,0.0,100
277,0.0,82
278,0.0,51
279,0.0,25
280,0.0,73
281,0.0,56
282,0.0,55
283,0.0,38
284,0.0,100
285,0.0,100
286,0.0,92
287,0.0,100
288,0.0,100
289,0.0,100
290,0.0,37
291,0.0,100
292,0.0,66
293,0.0,24
294,0.0,17
295,0.0,100
296,0.0,59
297,0.0,25
298,0.0,73
299,0.0,100
300,0.0,29
301,0.0,100
302,0.0,72
303,0.0,6
304,1.0,57
305,0.0,47
306,0.0,48
307,0.0,13
308,0.0,100
309,0.0,38
310,0.0,100
311,0.0,20
312,0.0,100
313,0.0,100
314,0.0,5
315,0.0,39
316,0.0,11
317,0.0,83
318,0.0,42
319,0.0,100
320,0.0,99
321,0.0,83
322,0.0,28
323,0.0,46
324,0.0,100
325,0.0,100
326,0.0,62
327,0.0,100
328,0.0,23
329,0.0,91
330,0.0,53
331,0.0,19
332,0.0,26
333,0.0,93
334,0.0,38
335,0.0,22
336,0.0,43
337,0.0,100
338,0.0,90
339,0.0,18
340,0.0,45
341,0.0,65
342,1.0,22
343,0.0,100
344,1.0,15
345,1.0,72
346,0.0,5
347,1.0,6
348,1.0,6
349,1.0,9
350,1.0,8
351,1.0,9
352,1.0,8
353,1.0,6
354,1.0,6
355,1.0,10
356,1.0,6
357,0.0,5
358,0.0,3
359,1.0,6
360,1.0,6
361,1.0,6
362,1.0,6
363,1.0,8
364,1.0,6
365,1.0,8
366,1.0,6
367,1.0,6
368,1.0,8
369,1.0,6
370,1.0,6
371,0.0,5
372,1.0,6
373,0.0,6
374,1.0,6
375,1.0,12
376,1.0,6
377,1.0,6
378,1.0,9
379,1.0,6
380,1.0,6
381,0.0,2
382,0.0,3
383,0.0,2
384,0.0,4
385,0.0,3
386,1.0,7
387,1.0,6
388,1.0,6
389,1.0,8
390,1.0,9
391,1.0,8
392,1.0,8
393,1.0,6
394,1.0,6
395,1.0,7
396,1.0,6
397,0.0,5
398,0.0,5
399,1.0,10
400,1.0,6
401,0.0,3
402,1.0,6
403,1.0,7
404,1.0,6
405,1.0,6
406,1.0,6
407,1.0,6
408,1.0,6
409,1.0,6
410,1.0,6
411,0.0,5
412,1.0,6
413,1.0,6
414,0.0,2
415,1.0,6
416,1.0,6
417,1.0,6
418,1.0,6
419,1.0,6
420,1.0,8
421,1.0,6
422,1.0,6
423,1.0,6
424,1.0,6
425,1.0,7
426,0.0,5
427,1.0,6
428,1.0,6
429,1.0,6
430,1.0,8
431,1.0,6
432,1.0,6
433,1.0,6
434,1.0,6
435,0.0,2
436,1.0,8
437,1.0,7
438,1.0,6
439,1.0,7
440,1.0,6
441,1.0,6
442,0.0,3
443,0.0,4
444,1.0,6
445,1.0,6
446,1.0,7
447,1.0,6
448,1.0,6
449,1.0,6
450,1.0,6
451,1.0,6
452,1.0,6
453,1.0,8
454,1.0,6
455,1.0,6
456,1.0,6
457,1.0,6
458,1.0,6
459,1.0,7
460,1.0,8
461,1.0,6
462,1.0,7
463,1.0,6
464,1.0,6
465,1.0,6
466,1.0,6
467,1.0,8
468,1.0,6
469,1.0,6
470,1.0,8
471,1.0,6
472,1.0,11
473,1.0,6
474,1.0,6
475,1.0,6
476,1.0,8
477,0.0,2
478,1.0,7
479,1.0,6
480,1.0,6
481,1.0,7
482,1.0,6
483,1.0,6
484,1.0,6
485,1.0,6
486,0.0,3
487,1.0,7
488,1.0,6
489,1.0,6
490,1.0,6
491,0.0,3
492,1.0,6
493,1.0,7
494,1.0,12
495,1.0,6
496,0.0,9
497,1.0,6
498,1.0,6
499,0.0,8
500,1.0,6
501,0.0,3
502,0.0,5
503,0.0,3
504,1.0,6
505,1.0,6
506,1.0,6
507,1.0,6
508,1.0,6
509,1.0,6
510,1.0,6
511,1.0,6
512,1.0,6
513,1.0,6
514,0.0,2
515,1.0,7
516,1.0,6
517,1.0,6
518,1.0,6
519,1.0,6
520,1.0,6
521,1.0,7
522,0.0,4
523,1.0,6
524,0.0,5
525,1.0,6
526,1.0,6
527,1.0,6
528,1.0,6
529,0.0,3
530,1.0,6
531,1.0,6
532,1.0,6
533,1.0,7
534,1.0,8
535,1.0,6
536,1.0,6
537,1.0,6
538,1.0,6
539,1.0,7
540,1.0,7
541,1.0,7
542,1.0,8
543,1.0,6
544,1.0,10
545,1.0,6
546,1.0,6
547,1.0,6
548,1.0,8
549,1.0,6
550,1.0,6
551,1.0,8
552,1.0,6
553,1.0,7
554,1.0,6
555,1.0,7
556,1.0,6
557,1.0,6
558,1.0,7
559,1.0,7
560,1.0,7
561,1.0,6
562,1.0,6
563,1.0,6
564,1.0,6
565,1.0,6
566,1.0,6
567,1.0,6
568,1.0,7
569,0.0,4
570,1.0,8
571,1.0,8
572,1.0,7
573,1.0,6
574,1.0,8
575,1.0,6
576,1.0,6
577,1.0,7
578,1.0,6
579,1.0,6
580,1.0,8
581,1.0,7
582,1.0,6
583,1.0,6
584,0.0,3
585,1.0,11
586,1.0,6
587,1.0,8
588,0.0,2
589,1.0,6
590,1.0,6
591,1.0,6
592,1.0,6
593,1.0,8
594,1.0,6
595,1.0,7
596,1.0,6
597,1.0,7
598,1.0,6
599,1.0,8
600,0.0,2
601,1.0,6
602,1.0,7
603,1.0,6
604,1.0,6
605,1.0,10
606,1.0,7
607,1.0,6
608,1.0,6
609,1.0,6
610,1.0,6
611,1.0,6
612,1.0,7
613,0.0,4
614,1.0,7
615,1.0,6
616,1.0,8
617,0.0,3
618,1.0,6
619,1.0,6
620,1.0,6
621,1.0,6
622,0.0,2
623,1.0,6
624,1.0,6
625,1.0,6
626,1.0,6
627,1.0,6
628,1.0,7
629,1.0,6
630,1.0,6
631,1.0,7
632,1.0,6
633,1.0,6
634,1.0,6
635,1.0,6
636,1.0,6
637,1.0,6
638,1.0,6
639,1.0,8
640,1.0,6
641,1.0,8
642,1.0,7
643,1.0,6
644,0.0,3
645,1.0,6
646,1.0,7
647,1.0,6
648,1.0,6
649,1.0,6
650,1.0,10
651,1.0,6
652,1.0,6
653,1.0,6
654,1.0,6
655,1.0,10
656,1.0,6
657,1.0,8
658,1.0,8
659,1.0,7
660,1.0,6
661,0.0,5
662,0.0,2
663,1.0,8
664,1.0,6
665,1.0,10
666,1.0,6
667,1.0,8
668,1.0,10
669,1.0,6
670,1.0,6
671,1.0,6
672,1.0,10
673,1.0,6
674,0.0,4
675,1.0,6
676,1.0,6
677,1.0,6
678,1.0,15
679,1.0,6
680,1.0,6
681,1.0,6
682,1.0,6
683,1.0,6
684,1.0,6
685,1.0,8
686,1.0,6
687,1.0,7
688,1.0,6
689,1.0,6
690,1.0,8
691,1.0,6
692,1.0,6
693,1.0,8
694,1.0,8
695,1.0,6
696,1.0,6
697,1.0,6
698,1.0,10
699,1.0,6
700,1.0,6
701,1.0,6
702,1.0,6
703,1.0,6
704,1.0,6
705,1.0,6
706,1.0,8
707,1.0,8
708,1.0,6
709,1.0,6
710,0.0,2
711,1.0,6
712,1.0,6
713,1.0,6
714,1.0,8
715,1.0,6
716,1.0,6
717,1.0,6
718,1.0,6
719,1.0,6
720,1.0,6
721,1.0,6
722,1.0,6
723,1.0,6
724,1.0,7
725,0.0,3
726,1.0,7
727,1.0,6
728,1.0,6
729,1.0,6
730,0.0,2
731,1.0,6
732,1.0,8
733,1.0,6
734,1.0,6
735,1.0,6
736,1.0,6
737,1.0,9
738,1.0,6
739,1.0,6
740,1.0,6
741,1.0,6
742,1.0,6
743,1.0,6
744,1.0,9
745,1.0,7
746,0.0,4
747,1.0,6
748,1.0,8
749,1.0,11
750,1.0,6
751,1.0,6
752,1.0,6
753,1.0,6
754,1.0,6
755,1.0,8
756,1.0,6
757,1.0,6
758,1.0,8
759,1.0,7
760,1.0,6
761,1.0,8
762,1.0,6
763,0.0,5
764,1.0,9
765,1.0,8
766,1.0,8
767,1.0,6
768,1.0,8
769,1.0,8
770,1.0,6
771,0.0,5
772,0.0,3
773,0.0,2
774,1.0,8
775,1.0,6
776,1.0,6
777,1.0,6
778,1.0,6
779,1.0,6
780,1.0,6
781,1.0,6
782,1.0,6
783,1.0,6
784,1.0,6
785,1.0,6
786,1.0,6
787,1.0,6
788,1.0,6
789,0.0,2
790,1.0,6
791,0.0,4
792,1.0,6
793,1.0,6
794,1.0,6
795,1.0,6
796,1.0,6
797,1.0,8
798,0.0,5
799,1.0,6
1 episodes rewards steps
2 0 0.0 20
3 1 0.0 14
4 2 0.0 13
5 3 0.0 9
6 4 0.0 10
7 5 0.0 6
8 6 0.0 11
9 7 0.0 6
10 8 0.0 3
11 9 0.0 9
12 10 0.0 11
13 11 0.0 22
14 12 0.0 5
15 13 0.0 16
16 14 0.0 4
17 15 0.0 9
18 16 0.0 18
19 17 0.0 2
20 18 0.0 4
21 19 0.0 8
22 20 0.0 7
23 21 0.0 4
24 22 0.0 22
25 23 0.0 15
26 24 0.0 5
27 25 0.0 16
28 26 0.0 7
29 27 0.0 19
30 28 0.0 22
31 29 0.0 16
32 30 0.0 11
33 31 0.0 22
34 32 0.0 28
35 33 0.0 23
36 34 0.0 4
37 35 0.0 11
38 36 0.0 8
39 37 0.0 15
40 38 0.0 5
41 39 0.0 7
42 40 0.0 9
43 41 0.0 4
44 42 0.0 3
45 43 0.0 6
46 44 0.0 41
47 45 0.0 9
48 46 0.0 23
49 47 0.0 3
50 48 1.0 38
51 49 0.0 29
52 50 0.0 17
53 51 0.0 4
54 52 0.0 2
55 53 0.0 25
56 54 0.0 6
57 55 0.0 2
58 56 0.0 30
59 57 0.0 6
60 58 0.0 7
61 59 0.0 11
62 60 0.0 9
63 61 0.0 8
64 62 0.0 23
65 63 0.0 10
66 64 0.0 3
67 65 0.0 5
68 66 0.0 7
69 67 0.0 18
70 68 0.0 8
71 69 0.0 26
72 70 0.0 6
73 71 0.0 14
74 72 0.0 4
75 73 0.0 25
76 74 0.0 21
77 75 0.0 13
78 76 0.0 4
79 77 0.0 29
80 78 0.0 21
81 79 0.0 6
82 80 0.0 6
83 81 0.0 11
84 82 0.0 21
85 83 0.0 9
86 84 0.0 9
87 85 0.0 7
88 86 0.0 48
89 87 0.0 23
90 88 0.0 100
91 89 0.0 60
92 90 0.0 7
93 91 0.0 10
94 92 0.0 24
95 93 0.0 4
96 94 0.0 7
97 95 0.0 17
98 96 0.0 87
99 97 0.0 28
100 98 0.0 7
101 99 0.0 5
102 100 0.0 12
103 101 0.0 14
104 102 0.0 6
105 103 0.0 13
106 104 0.0 93
107 105 0.0 4
108 106 0.0 50
109 107 0.0 8
110 108 0.0 12
111 109 0.0 43
112 110 0.0 30
113 111 0.0 15
114 112 0.0 19
115 113 0.0 100
116 114 0.0 82
117 115 0.0 40
118 116 0.0 88
119 117 0.0 19
120 118 0.0 30
121 119 0.0 27
122 120 0.0 5
123 121 0.0 87
124 122 0.0 9
125 123 0.0 64
126 124 0.0 27
127 125 0.0 68
128 126 0.0 81
129 127 0.0 86
130 128 0.0 100
131 129 0.0 100
132 130 0.0 27
133 131 0.0 41
134 132 0.0 70
135 133 0.0 27
136 134 0.0 6
137 135 0.0 18
138 136 0.0 38
139 137 0.0 26
140 138 0.0 36
141 139 0.0 3
142 140 0.0 61
143 141 0.0 100
144 142 0.0 4
145 143 0.0 39
146 144 0.0 18
147 145 0.0 33
148 146 0.0 29
149 147 0.0 49
150 148 0.0 88
151 149 0.0 22
152 150 0.0 65
153 151 0.0 36
154 152 0.0 30
155 153 0.0 58
156 154 0.0 43
157 155 0.0 53
158 156 0.0 43
159 157 0.0 13
160 158 0.0 8
161 159 0.0 39
162 160 0.0 29
163 161 0.0 26
164 162 0.0 60
165 163 0.0 100
166 164 0.0 31
167 165 0.0 22
168 166 0.0 100
169 167 0.0 46
170 168 0.0 23
171 169 0.0 54
172 170 0.0 8
173 171 0.0 58
174 172 0.0 3
175 173 0.0 47
176 174 0.0 16
177 175 0.0 21
178 176 0.0 44
179 177 0.0 29
180 178 0.0 100
181 179 0.0 100
182 180 0.0 62
183 181 0.0 83
184 182 0.0 26
185 183 0.0 24
186 184 0.0 10
187 185 0.0 12
188 186 0.0 40
189 187 0.0 25
190 188 0.0 18
191 189 0.0 60
192 190 0.0 100
193 191 0.0 100
194 192 0.0 24
195 193 0.0 56
196 194 0.0 71
197 195 0.0 19
198 196 0.0 100
199 197 0.0 44
200 198 0.0 41
201 199 0.0 41
202 200 0.0 60
203 201 0.0 31
204 202 0.0 34
205 203 0.0 35
206 204 0.0 59
207 205 0.0 51
208 206 0.0 100
209 207 0.0 100
210 208 0.0 100
211 209 0.0 100
212 210 0.0 37
213 211 0.0 68
214 212 0.0 40
215 213 0.0 17
216 214 0.0 79
217 215 0.0 100
218 216 0.0 26
219 217 0.0 61
220 218 0.0 25
221 219 0.0 18
222 220 0.0 27
223 221 0.0 13
224 222 0.0 100
225 223 0.0 87
226 224 0.0 100
227 225 0.0 92
228 226 0.0 100
229 227 0.0 8
230 228 0.0 100
231 229 0.0 64
232 230 0.0 17
233 231 0.0 82
234 232 0.0 100
235 233 0.0 94
236 234 0.0 7
237 235 0.0 36
238 236 0.0 100
239 237 0.0 56
240 238 0.0 17
241 239 0.0 100
242 240 0.0 83
243 241 0.0 100
244 242 0.0 100
245 243 0.0 43
246 244 0.0 87
247 245 0.0 42
248 246 0.0 80
249 247 0.0 54
250 248 0.0 82
251 249 0.0 97
252 250 0.0 65
253 251 0.0 83
254 252 0.0 100
255 253 0.0 59
256 254 0.0 100
257 255 0.0 78
258 256 0.0 100
259 257 0.0 100
260 258 0.0 43
261 259 0.0 80
262 260 0.0 100
263 261 0.0 70
264 262 0.0 94
265 263 0.0 100
266 264 0.0 100
267 265 0.0 37
268 266 0.0 11
269 267 0.0 31
270 268 0.0 100
271 269 0.0 34
272 270 0.0 32
273 271 0.0 58
274 272 0.0 38
275 273 0.0 28
276 274 0.0 100
277 275 0.0 59
278 276 0.0 100
279 277 0.0 82
280 278 0.0 51
281 279 0.0 25
282 280 0.0 73
283 281 0.0 56
284 282 0.0 55
285 283 0.0 38
286 284 0.0 100
287 285 0.0 100
288 286 0.0 92
289 287 0.0 100
290 288 0.0 100
291 289 0.0 100
292 290 0.0 37
293 291 0.0 100
294 292 0.0 66
295 293 0.0 24
296 294 0.0 17
297 295 0.0 100
298 296 0.0 59
299 297 0.0 25
300 298 0.0 73
301 299 0.0 100
302 300 0.0 29
303 301 0.0 100
304 302 0.0 72
305 303 0.0 6
306 304 1.0 57
307 305 0.0 47
308 306 0.0 48
309 307 0.0 13
310 308 0.0 100
311 309 0.0 38
312 310 0.0 100
313 311 0.0 20
314 312 0.0 100
315 313 0.0 100
316 314 0.0 5
317 315 0.0 39
318 316 0.0 11
319 317 0.0 83
320 318 0.0 42
321 319 0.0 100
322 320 0.0 99
323 321 0.0 83
324 322 0.0 28
325 323 0.0 46
326 324 0.0 100
327 325 0.0 100
328 326 0.0 62
329 327 0.0 100
330 328 0.0 23
331 329 0.0 91
332 330 0.0 53
333 331 0.0 19
334 332 0.0 26
335 333 0.0 93
336 334 0.0 38
337 335 0.0 22
338 336 0.0 43
339 337 0.0 100
340 338 0.0 90
341 339 0.0 18
342 340 0.0 45
343 341 0.0 65
344 342 1.0 22
345 343 0.0 100
346 344 1.0 15
347 345 1.0 72
348 346 0.0 5
349 347 1.0 6
350 348 1.0 6
351 349 1.0 9
352 350 1.0 8
353 351 1.0 9
354 352 1.0 8
355 353 1.0 6
356 354 1.0 6
357 355 1.0 10
358 356 1.0 6
359 357 0.0 5
360 358 0.0 3
361 359 1.0 6
362 360 1.0 6
363 361 1.0 6
364 362 1.0 6
365 363 1.0 8
366 364 1.0 6
367 365 1.0 8
368 366 1.0 6
369 367 1.0 6
370 368 1.0 8
371 369 1.0 6
372 370 1.0 6
373 371 0.0 5
374 372 1.0 6
375 373 0.0 6
376 374 1.0 6
377 375 1.0 12
378 376 1.0 6
379 377 1.0 6
380 378 1.0 9
381 379 1.0 6
382 380 1.0 6
383 381 0.0 2
384 382 0.0 3
385 383 0.0 2
386 384 0.0 4
387 385 0.0 3
388 386 1.0 7
389 387 1.0 6
390 388 1.0 6
391 389 1.0 8
392 390 1.0 9
393 391 1.0 8
394 392 1.0 8
395 393 1.0 6
396 394 1.0 6
397 395 1.0 7
398 396 1.0 6
399 397 0.0 5
400 398 0.0 5
401 399 1.0 10
402 400 1.0 6
403 401 0.0 3
404 402 1.0 6
405 403 1.0 7
406 404 1.0 6
407 405 1.0 6
408 406 1.0 6
409 407 1.0 6
410 408 1.0 6
411 409 1.0 6
412 410 1.0 6
413 411 0.0 5
414 412 1.0 6
415 413 1.0 6
416 414 0.0 2
417 415 1.0 6
418 416 1.0 6
419 417 1.0 6
420 418 1.0 6
421 419 1.0 6
422 420 1.0 8
423 421 1.0 6
424 422 1.0 6
425 423 1.0 6
426 424 1.0 6
427 425 1.0 7
428 426 0.0 5
429 427 1.0 6
430 428 1.0 6
431 429 1.0 6
432 430 1.0 8
433 431 1.0 6
434 432 1.0 6
435 433 1.0 6
436 434 1.0 6
437 435 0.0 2
438 436 1.0 8
439 437 1.0 7
440 438 1.0 6
441 439 1.0 7
442 440 1.0 6
443 441 1.0 6
444 442 0.0 3
445 443 0.0 4
446 444 1.0 6
447 445 1.0 6
448 446 1.0 7
449 447 1.0 6
450 448 1.0 6
451 449 1.0 6
452 450 1.0 6
453 451 1.0 6
454 452 1.0 6
455 453 1.0 8
456 454 1.0 6
457 455 1.0 6
458 456 1.0 6
459 457 1.0 6
460 458 1.0 6
461 459 1.0 7
462 460 1.0 8
463 461 1.0 6
464 462 1.0 7
465 463 1.0 6
466 464 1.0 6
467 465 1.0 6
468 466 1.0 6
469 467 1.0 8
470 468 1.0 6
471 469 1.0 6
472 470 1.0 8
473 471 1.0 6
474 472 1.0 11
475 473 1.0 6
476 474 1.0 6
477 475 1.0 6
478 476 1.0 8
479 477 0.0 2
480 478 1.0 7
481 479 1.0 6
482 480 1.0 6
483 481 1.0 7
484 482 1.0 6
485 483 1.0 6
486 484 1.0 6
487 485 1.0 6
488 486 0.0 3
489 487 1.0 7
490 488 1.0 6
491 489 1.0 6
492 490 1.0 6
493 491 0.0 3
494 492 1.0 6
495 493 1.0 7
496 494 1.0 12
497 495 1.0 6
498 496 0.0 9
499 497 1.0 6
500 498 1.0 6
501 499 0.0 8
502 500 1.0 6
503 501 0.0 3
504 502 0.0 5
505 503 0.0 3
506 504 1.0 6
507 505 1.0 6
508 506 1.0 6
509 507 1.0 6
510 508 1.0 6
511 509 1.0 6
512 510 1.0 6
513 511 1.0 6
514 512 1.0 6
515 513 1.0 6
516 514 0.0 2
517 515 1.0 7
518 516 1.0 6
519 517 1.0 6
520 518 1.0 6
521 519 1.0 6
522 520 1.0 6
523 521 1.0 7
524 522 0.0 4
525 523 1.0 6
526 524 0.0 5
527 525 1.0 6
528 526 1.0 6
529 527 1.0 6
530 528 1.0 6
531 529 0.0 3
532 530 1.0 6
533 531 1.0 6
534 532 1.0 6
535 533 1.0 7
536 534 1.0 8
537 535 1.0 6
538 536 1.0 6
539 537 1.0 6
540 538 1.0 6
541 539 1.0 7
542 540 1.0 7
543 541 1.0 7
544 542 1.0 8
545 543 1.0 6
546 544 1.0 10
547 545 1.0 6
548 546 1.0 6
549 547 1.0 6
550 548 1.0 8
551 549 1.0 6
552 550 1.0 6
553 551 1.0 8
554 552 1.0 6
555 553 1.0 7
556 554 1.0 6
557 555 1.0 7
558 556 1.0 6
559 557 1.0 6
560 558 1.0 7
561 559 1.0 7
562 560 1.0 7
563 561 1.0 6
564 562 1.0 6
565 563 1.0 6
566 564 1.0 6
567 565 1.0 6
568 566 1.0 6
569 567 1.0 6
570 568 1.0 7
571 569 0.0 4
572 570 1.0 8
573 571 1.0 8
574 572 1.0 7
575 573 1.0 6
576 574 1.0 8
577 575 1.0 6
578 576 1.0 6
579 577 1.0 7
580 578 1.0 6
581 579 1.0 6
582 580 1.0 8
583 581 1.0 7
584 582 1.0 6
585 583 1.0 6
586 584 0.0 3
587 585 1.0 11
588 586 1.0 6
589 587 1.0 8
590 588 0.0 2
591 589 1.0 6
592 590 1.0 6
593 591 1.0 6
594 592 1.0 6
595 593 1.0 8
596 594 1.0 6
597 595 1.0 7
598 596 1.0 6
599 597 1.0 7
600 598 1.0 6
601 599 1.0 8
602 600 0.0 2
603 601 1.0 6
604 602 1.0 7
605 603 1.0 6
606 604 1.0 6
607 605 1.0 10
608 606 1.0 7
609 607 1.0 6
610 608 1.0 6
611 609 1.0 6
612 610 1.0 6
613 611 1.0 6
614 612 1.0 7
615 613 0.0 4
616 614 1.0 7
617 615 1.0 6
618 616 1.0 8
619 617 0.0 3
620 618 1.0 6
621 619 1.0 6
622 620 1.0 6
623 621 1.0 6
624 622 0.0 2
625 623 1.0 6
626 624 1.0 6
627 625 1.0 6
628 626 1.0 6
629 627 1.0 6
630 628 1.0 7
631 629 1.0 6
632 630 1.0 6
633 631 1.0 7
634 632 1.0 6
635 633 1.0 6
636 634 1.0 6
637 635 1.0 6
638 636 1.0 6
639 637 1.0 6
640 638 1.0 6
641 639 1.0 8
642 640 1.0 6
643 641 1.0 8
644 642 1.0 7
645 643 1.0 6
646 644 0.0 3
647 645 1.0 6
648 646 1.0 7
649 647 1.0 6
650 648 1.0 6
651 649 1.0 6
652 650 1.0 10
653 651 1.0 6
654 652 1.0 6
655 653 1.0 6
656 654 1.0 6
657 655 1.0 10
658 656 1.0 6
659 657 1.0 8
660 658 1.0 8
661 659 1.0 7
662 660 1.0 6
663 661 0.0 5
664 662 0.0 2
665 663 1.0 8
666 664 1.0 6
667 665 1.0 10
668 666 1.0 6
669 667 1.0 8
670 668 1.0 10
671 669 1.0 6
672 670 1.0 6
673 671 1.0 6
674 672 1.0 10
675 673 1.0 6
676 674 0.0 4
677 675 1.0 6
678 676 1.0 6
679 677 1.0 6
680 678 1.0 15
681 679 1.0 6
682 680 1.0 6
683 681 1.0 6
684 682 1.0 6
685 683 1.0 6
686 684 1.0 6
687 685 1.0 8
688 686 1.0 6
689 687 1.0 7
690 688 1.0 6
691 689 1.0 6
692 690 1.0 8
693 691 1.0 6
694 692 1.0 6
695 693 1.0 8
696 694 1.0 8
697 695 1.0 6
698 696 1.0 6
699 697 1.0 6
700 698 1.0 10
701 699 1.0 6
702 700 1.0 6
703 701 1.0 6
704 702 1.0 6
705 703 1.0 6
706 704 1.0 6
707 705 1.0 6
708 706 1.0 8
709 707 1.0 8
710 708 1.0 6
711 709 1.0 6
712 710 0.0 2
713 711 1.0 6
714 712 1.0 6
715 713 1.0 6
716 714 1.0 8
717 715 1.0 6
718 716 1.0 6
719 717 1.0 6
720 718 1.0 6
721 719 1.0 6
722 720 1.0 6
723 721 1.0 6
724 722 1.0 6
725 723 1.0 6
726 724 1.0 7
727 725 0.0 3
728 726 1.0 7
729 727 1.0 6
730 728 1.0 6
731 729 1.0 6
732 730 0.0 2
733 731 1.0 6
734 732 1.0 8
735 733 1.0 6
736 734 1.0 6
737 735 1.0 6
738 736 1.0 6
739 737 1.0 9
740 738 1.0 6
741 739 1.0 6
742 740 1.0 6
743 741 1.0 6
744 742 1.0 6
745 743 1.0 6
746 744 1.0 9
747 745 1.0 7
748 746 0.0 4
749 747 1.0 6
750 748 1.0 8
751 749 1.0 11
752 750 1.0 6
753 751 1.0 6
754 752 1.0 6
755 753 1.0 6
756 754 1.0 6
757 755 1.0 8
758 756 1.0 6
759 757 1.0 6
760 758 1.0 8
761 759 1.0 7
762 760 1.0 6
763 761 1.0 8
764 762 1.0 6
765 763 0.0 5
766 764 1.0 9
767 765 1.0 8
768 766 1.0 8
769 767 1.0 6
770 768 1.0 8
771 769 1.0 8
772 770 1.0 6
773 771 0.0 5
774 772 0.0 3
775 773 0.0 2
776 774 1.0 8
777 775 1.0 6
778 776 1.0 6
779 777 1.0 6
780 778 1.0 6
781 779 1.0 6
782 780 1.0 6
783 781 1.0 6
784 782 1.0 6
785 783 1.0 6
786 784 1.0 6
787 785 1.0 6
788 786 1.0 6
789 787 1.0 6
790 788 1.0 6
791 789 0.0 2
792 790 1.0 6
793 791 0.0 4
794 792 1.0 6
795 793 1.0 6
796 794 1.0 6
797 795 1.0 6
798 796 1.0 6
799 797 1.0 8
800 798 0.0 5
801 799 1.0 6

BIN
projects/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/models/Qleaning_model.pkl Normal file
View File

Binary file not shown.

6
projects/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/results/params.json → projects/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/results/params.json
View File

@@ -1,6 +1,6 @@
{
"algo_name": "Q-learning",
"env_name": "FrozenLake-v1",
"env_name": "FrozenLakeNoSlippery-v1",
"train_eps": 800,
"test_eps": 20,
"gamma": 0.9,
@@ -12,8 +12,8 @@
"seed": 10,
"show_fig": false,
"save_fig": true,
"result_path": "/Users/jj/Desktop/rl-tutorials/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/results/",
"model_path": "/Users/jj/Desktop/rl-tutorials/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/models/",
"result_path": "/Users/jj/Desktop/rl-tutorials/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/results/",
"model_path": "/Users/jj/Desktop/rl-tutorials/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/models/",
"n_states": 16,
"n_actions": 4
}

BIN
projects/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/results/testing_curve.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 24 KiB

0
projects/codes/QLearning/outputs/FrozenLake-v1/20220824-112735/results/testing_results.csv → projects/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/results/testing_results.csv
View File

BIN
projects/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/results/training_curve.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 55 KiB

801
projects/codes/QLearning/outputs/FrozenLakeNoSlippery-v1/20220825-114335/results/training_results.csv Normal file
View File

@@ -0,0 +1,801 @@
episodes,rewards,steps
0,0.0,20
1,0.0,14
2,0.0,13
3,0.0,9
4,0.0,10
5,0.0,6
6,0.0,11
7,0.0,6
8,0.0,3
9,0.0,9
10,0.0,11
11,0.0,22
12,0.0,5
13,0.0,16
14,0.0,4
15,0.0,9
16,0.0,18
17,0.0,2
18,0.0,4
19,0.0,8
20,0.0,7
21,0.0,4
22,0.0,22
23,0.0,15
24,0.0,5
25,0.0,16
26,0.0,7
27,0.0,19
28,0.0,22
29,0.0,16
30,0.0,11
31,0.0,22
32,0.0,28
33,0.0,23
34,0.0,4
35,0.0,11
36,0.0,8
37,0.0,15
38,0.0,5
39,0.0,7
40,0.0,9
41,0.0,4
42,0.0,3
43,0.0,6
44,0.0,41
45,0.0,9
46,0.0,23
47,0.0,3
48,1.0,38
49,0.0,29
50,0.0,17
51,0.0,4
52,0.0,2
53,0.0,25
54,0.0,6
55,0.0,2
56,0.0,30
57,0.0,6
58,0.0,7
59,0.0,11
60,0.0,9
61,0.0,8
62,0.0,23
63,0.0,10
64,0.0,3
65,0.0,5
66,0.0,7
67,0.0,18
68,0.0,8
69,0.0,26
70,0.0,6
71,0.0,14
72,0.0,4
73,0.0,25
74,0.0,21
75,0.0,13
76,0.0,4
77,0.0,29
78,0.0,21
79,0.0,6
80,0.0,6
81,0.0,11
82,0.0,21
83,0.0,9
84,0.0,9
85,0.0,7
86,0.0,48
87,0.0,23
88,0.0,160
89,0.0,7
90,0.0,10
91,0.0,24
92,0.0,4
93,0.0,7
94,0.0,17
95,0.0,87
96,0.0,28
97,0.0,7
98,0.0,5
99,0.0,12
100,0.0,14
101,0.0,6
102,0.0,13
103,0.0,93
104,0.0,4
105,0.0,50
106,0.0,8
107,0.0,12
108,0.0,43
109,0.0,30
110,0.0,15
111,0.0,19
112,0.0,182
113,0.0,40
114,0.0,88
115,0.0,19
116,0.0,30
117,0.0,27
118,0.0,5
119,0.0,87
120,0.0,9
121,0.0,64
122,0.0,27
123,0.0,68
124,0.0,81
125,0.0,86
126,0.0,227
127,0.0,41
128,0.0,70
129,0.0,27
130,0.0,6
131,0.0,18
132,0.0,38
133,0.0,26
134,0.0,36
135,0.0,3
136,0.0,61
137,0.0,105
138,0.0,38
139,0.0,18
140,0.0,33
141,0.0,29
142,0.0,49
143,0.0,88
144,0.0,22
145,0.0,65
146,0.0,36
147,0.0,30
148,0.0,58
149,0.0,43
150,0.0,53
151,0.0,43
152,0.0,13
153,0.0,8
154,0.0,39
155,0.0,29
156,0.0,26
157,0.0,60
158,0.0,153
159,0.0,116
160,0.0,53
161,0.0,54
162,0.0,8
163,0.0,58
164,0.0,3
165,0.0,47
166,0.0,16
167,0.0,21
168,0.0,44
169,0.0,29
170,0.0,104
171,0.0,158
172,0.0,83
173,0.0,26
174,0.0,24
175,0.0,10
176,0.0,12
177,0.0,40
178,0.0,25
179,0.0,18
180,0.0,60
181,0.0,203
182,0.0,23
183,0.0,54
184,0.0,71
185,0.0,19
186,0.0,118
187,0.0,26
188,0.0,41
189,0.0,41
190,0.0,60
191,0.0,31
192,0.0,34
193,0.0,35
194,0.0,59
195,0.0,51
196,0.0,426
197,0.0,79
198,0.0,40
199,0.0,17
200,0.0,79
201,0.0,126
202,0.0,61
203,0.0,25
204,0.0,18
205,0.0,27
206,0.0,13
207,0.0,187
208,0.0,160
209,0.0,32
210,0.0,108
211,0.0,164
212,0.0,17
213,0.0,82
214,0.0,194
215,0.0,7
216,0.0,36
217,0.0,156
218,0.0,17
219,0.0,183
220,0.0,243
221,0.0,87
222,0.0,42
223,0.0,80
224,0.0,54
225,0.0,82
226,0.0,97
227,0.0,65
228,0.0,83
229,0.0,159
230,0.0,178
231,0.0,104
232,0.0,21
233,0.0,118
234,0.0,80
235,0.0,170
236,0.0,94
237,0.0,235
238,0.0,13
239,0.0,31
240,0.0,134
241,0.0,32
242,0.0,58
243,0.0,38
244,0.0,28
245,0.0,159
246,0.0,182
247,0.0,51
248,0.0,25
249,0.0,73
250,0.0,56
251,0.0,55
252,0.0,38
253,0.0,292
254,0.0,319
255,0.0,100
256,0.0,84
257,0.0,24
258,0.0,17
259,0.0,159
260,0.0,25
261,0.0,73
262,0.0,130
263,0.0,111
264,0.0,65
265,1.0,58
266,0.0,47
267,0.0,48
268,0.0,13
269,0.0,100
270,0.0,38
271,0.0,111
272,0.0,226
273,0.0,38
274,0.0,83
275,0.0,42
276,0.0,199
277,0.0,83
278,0.0,28
279,0.0,46
280,0.0,262
281,0.0,123
282,0.0,91
283,0.0,53
284,0.0,19
285,0.0,26
286,0.0,93
287,0.0,38
288,0.0,22
289,0.0,43
290,0.0,163
291,0.0,25
292,0.0,59
293,0.0,71
294,0.0,20
295,0.0,115
296,0.0,248
297,0.0,66
298,0.0,58
299,0.0,129
300,0.0,122
301,0.0,47
302,0.0,60
303,0.0,79
304,1.0,137
305,0.0,27
306,1.0,93
307,0.0,46
308,1.0,83
309,1.0,8
310,1.0,6
311,1.0,6
312,0.0,4
313,1.0,6
314,0.0,2
315,1.0,6
316,1.0,6
317,1.0,6
318,1.0,6
319,1.0,8
320,0.0,5
321,1.0,6
322,1.0,7
323,0.0,5
324,1.0,6
325,1.0,6
326,1.0,8
327,1.0,6
328,1.0,6
329,1.0,6
330,1.0,7
331,1.0,6
332,1.0,6
333,0.0,3
334,1.0,7
335,0.0,4
336,1.0,6
337,1.0,6
338,1.0,7
339,1.0,6
340,1.0,6
341,1.0,7
342,1.0,7
343,1.0,7
344,1.0,6
345,1.0,6
346,1.0,6
347,1.0,6
348,1.0,6
349,1.0,6
350,1.0,6
351,1.0,7
352,0.0,4
353,1.0,8
354,1.0,8
355,1.0,7
356,1.0,6
357,1.0,8
358,1.0,6
359,1.0,6
360,1.0,7
361,1.0,6
362,1.0,6
363,1.0,8
364,1.0,7
365,1.0,6
366,1.0,6
367,0.0,3
368,1.0,11
369,1.0,6
370,1.0,8
371,0.0,2
372,1.0,6
373,1.0,6
374,1.0,6
375,1.0,6
376,1.0,8
377,1.0,6
378,1.0,7
379,1.0,6
380,1.0,7
381,1.0,6
382,1.0,8
383,0.0,2
384,1.0,6
385,1.0,7
386,1.0,6
387,1.0,6
388,1.0,10
389,1.0,7
390,1.0,6
391,1.0,6
392,1.0,6
393,1.0,6
394,1.0,6
395,1.0,7
396,0.0,4
397,1.0,7
398,1.0,6
399,1.0,8
400,0.0,3
401,1.0,6
402,1.0,6
403,1.0,6
404,1.0,6
405,0.0,2
406,1.0,6
407,1.0,6
408,1.0,6
409,1.0,6
410,1.0,6
411,1.0,7
412,1.0,6
413,1.0,6
414,1.0,7
415,1.0,6
416,1.0,6
417,1.0,6
418,1.0,6
419,1.0,6
420,1.0,6
421,1.0,6
422,1.0,8
423,1.0,6
424,1.0,8
425,1.0,7
426,1.0,6
427,0.0,3
428,1.0,6
429,1.0,7
430,1.0,6
431,1.0,6
432,1.0,6
433,1.0,10
434,1.0,6
435,1.0,6
436,1.0,6
437,1.0,6
438,1.0,10
439,1.0,6
440,1.0,8
441,1.0,8
442,1.0,7
443,1.0,6
444,0.0,5
445,0.0,2
446,1.0,8
447,1.0,6
448,1.0,10
449,1.0,6
450,1.0,8
451,1.0,10
452,1.0,6
453,1.0,6
454,1.0,6
455,1.0,10
456,1.0,6
457,0.0,4
458,1.0,6
459,1.0,6
460,1.0,6
461,1.0,15
462,1.0,6
463,1.0,6
464,1.0,6
465,1.0,6
466,1.0,6
467,1.0,6
468,1.0,8
469,1.0,6
470,1.0,7
471,1.0,6
472,1.0,6
473,1.0,8
474,1.0,6
475,1.0,6
476,1.0,8
477,1.0,8
478,1.0,6
479,1.0,6
480,1.0,6
481,1.0,10
482,1.0,6
483,1.0,6
484,1.0,6
485,1.0,6
486,1.0,6
487,1.0,6
488,1.0,6
489,1.0,8
490,1.0,8
491,1.0,6
492,1.0,6
493,0.0,2
494,1.0,6
495,1.0,6
496,1.0,6
497,1.0,8
498,1.0,6
499,1.0,6
500,1.0,6
501,1.0,6
502,1.0,6
503,1.0,6
504,1.0,6
505,1.0,6
506,1.0,6
507,1.0,7
508,0.0,3
509,1.0,7
510,1.0,6
511,1.0,6
512,1.0,6
513,0.0,2
514,1.0,6
515,1.0,8
516,1.0,6
517,1.0,6
518,1.0,6
519,1.0,6
520,1.0,9
521,1.0,6
522,1.0,6
523,1.0,6
524,1.0,6
525,1.0,6
526,1.0,6
527,1.0,9
528,1.0,7
529,0.0,4
530,1.0,6
531,1.0,8
532,1.0,11
533,1.0,6
534,1.0,6
535,1.0,6
536,1.0,6
537,1.0,6
538,1.0,8
539,1.0,6
540,1.0,6
541,1.0,8
542,1.0,7
543,1.0,6
544,1.0,8
545,1.0,6
546,0.0,5
547,1.0,9
548,1.0,8
549,1.0,8
550,1.0,6
551,1.0,8
552,1.0,8
553,1.0,6
554,0.0,5
555,0.0,3
556,0.0,2
557,1.0,8
558,1.0,6
559,1.0,6
560,1.0,6
561,1.0,6
562,1.0,6
563,1.0,6
564,1.0,6
565,1.0,6
566,1.0,6
567,1.0,6
568,1.0,6
569,1.0,6
570,1.0,6
571,1.0,6
572,0.0,2
573,1.0,6
574,0.0,4
575,1.0,6
576,1.0,6
577,1.0,6
578,1.0,6
579,1.0,6
580,1.0,8
581,0.0,5
582,1.0,6
583,1.0,6
584,1.0,6
585,1.0,6
586,1.0,6
587,1.0,6
588,0.0,3
589,1.0,6
590,1.0,6
591,1.0,6
592,0.0,2
593,1.0,6
594,0.0,4
595,1.0,6
596,1.0,6
597,1.0,6
598,1.0,6
599,1.0,8
600,1.0,6
601,1.0,7
602,1.0,6
603,1.0,7
604,1.0,6
605,0.0,2
606,1.0,6
607,1.0,6
608,0.0,5
609,0.0,3
610,0.0,3
611,1.0,6
612,0.0,5
613,1.0,8
614,1.0,8
615,1.0,6
616,1.0,6
617,1.0,7
618,1.0,6
619,1.0,6
620,1.0,6
621,1.0,6
622,1.0,6
623,1.0,8
624,0.0,2
625,1.0,6
626,1.0,6
627,1.0,6
628,1.0,6
629,1.0,6
630,1.0,6
631,1.0,6
632,1.0,8
633,1.0,6
634,1.0,8
635,1.0,6
636,1.0,6
637,1.0,8
638,1.0,8
639,0.0,5
640,0.0,4
641,0.0,4
642,1.0,6
643,1.0,6
644,1.0,6
645,1.0,6
646,1.0,8
647,1.0,6
648,0.0,4
649,1.0,6
650,1.0,8
651,1.0,6
652,1.0,6
653,1.0,6
654,1.0,6
655,1.0,6
656,1.0,6
657,1.0,6
658,1.0,8
659,1.0,8
660,1.0,6
661,1.0,8
662,1.0,9
663,1.0,6
664,1.0,6
665,1.0,6
666,1.0,6
667,1.0,10
668,1.0,6
669,1.0,6
670,1.0,6
671,1.0,11
672,1.0,10
673,1.0,8
674,1.0,6
675,1.0,6
676,1.0,6
677,0.0,5
678,1.0,6
679,0.0,2
680,1.0,9
681,1.0,6
682,1.0,8
683,1.0,7
684,1.0,6
685,1.0,6
686,1.0,7
687,0.0,3
688,1.0,7
689,0.0,2
690,1.0,6
691,1.0,6
692,1.0,8
693,1.0,8
694,1.0,6
695,1.0,6
696,0.0,2
697,1.0,8
698,1.0,6
699,1.0,8
700,1.0,6
701,1.0,6
702,1.0,9
703,1.0,6
704,1.0,8
705,1.0,11
706,1.0,6
707,1.0,6
708,1.0,6
709,1.0,6
710,1.0,8
711,1.0,6
712,1.0,6
713,1.0,6
714,0.0,5
715,1.0,6
716,1.0,6
717,1.0,6
718,1.0,6
719,1.0,6
720,1.0,7
721,1.0,6
722,1.0,6
723,1.0,6
724,1.0,6
725,1.0,10
726,1.0,6
727,1.0,6
728,1.0,6
729,1.0,6
730,1.0,6
731,1.0,7
732,1.0,6
733,1.0,8
734,1.0,7
735,1.0,6
736,1.0,6
737,1.0,14
738,1.0,6
739,1.0,6
740,1.0,12
741,1.0,6
742,1.0,6
743,1.0,6
744,1.0,6
745,1.0,6
746,1.0,6
747,0.0,3
748,1.0,6
749,1.0,6
750,1.0,6
751,1.0,7
752,1.0,6
753,1.0,6
754,1.0,6
755,1.0,8
756,0.0,2
757,1.0,6
758,1.0,6
759,1.0,6
760,1.0,6
761,1.0,6
762,1.0,6
763,1.0,6
764,1.0,6
765,1.0,6
766,0.0,4
767,1.0,8
768,1.0,6
769,0.0,2
770,1.0,10
771,1.0,8
772,1.0,6
773,1.0,6
774,1.0,6
775,0.0,3
776,1.0,6
777,1.0,6
778,0.0,6
779,1.0,8
780,1.0,6
781,1.0,9
782,1.0,6
783,1.0,6
784,1.0,8
785,1.0,8
786,1.0,6
787,0.0,5
788,1.0,6
789,1.0,6
790,1.0,6
791,1.0,6
792,1.0,6
793,1.0,6
794,1.0,8
795,1.0,6
796,0.0,2
797,1.0,8
798,1.0,7
799,1.0,6
1 episodes rewards steps
2 0 0.0 20
3 1 0.0 14
4 2 0.0 13
5 3 0.0 9
6 4 0.0 10
7 5 0.0 6
8 6 0.0 11
9 7 0.0 6
10 8 0.0 3
11 9 0.0 9
12 10 0.0 11
13 11 0.0 22
14 12 0.0 5
15 13 0.0 16
16 14 0.0 4
17 15 0.0 9
18 16 0.0 18
19 17 0.0 2
20 18 0.0 4
21 19 0.0 8
22 20 0.0 7
23 21 0.0 4
24 22 0.0 22
25 23 0.0 15
26 24 0.0 5
27 25 0.0 16
28 26 0.0 7
29 27 0.0 19
30 28 0.0 22
31 29 0.0 16
32 30 0.0 11
33 31 0.0 22
34 32 0.0 28
35 33 0.0 23
36 34 0.0 4
37 35 0.0 11
38 36 0.0 8
39 37 0.0 15
40 38 0.0 5
41 39 0.0 7
42 40 0.0 9
43 41 0.0 4
44 42 0.0 3
45 43 0.0 6
46 44 0.0 41
47 45 0.0 9
48 46 0.0 23
49 47 0.0 3
50 48 1.0 38
51 49 0.0 29
52 50 0.0 17
53 51 0.0 4
54 52 0.0 2
55 53 0.0 25
56 54 0.0 6
57 55 0.0 2
58 56 0.0 30
59 57 0.0 6
60 58 0.0 7
61 59 0.0 11
62 60 0.0 9
63 61 0.0 8
64 62 0.0 23
65 63 0.0 10
66 64 0.0 3
67 65 0.0 5
68 66 0.0 7
69 67 0.0 18
70 68 0.0 8
71 69 0.0 26
72 70 0.0 6
73 71 0.0 14
74 72 0.0 4
75 73 0.0 25
76 74 0.0 21
77 75 0.0 13
78 76 0.0 4
79 77 0.0 29
80 78 0.0 21
81 79 0.0 6
82 80 0.0 6
83 81 0.0 11
84 82 0.0 21
85 83 0.0 9
86 84 0.0 9
87 85 0.0 7
88 86 0.0 48
89 87 0.0 23
90 88 0.0 160
91 89 0.0 7
92 90 0.0 10
93 91 0.0 24
94 92 0.0 4
95 93 0.0 7
96 94 0.0 17
97 95 0.0 87
98 96 0.0 28
99 97 0.0 7
100 98 0.0 5
101 99 0.0 12
102 100 0.0 14
103 101 0.0 6
104 102 0.0 13
105 103 0.0 93
106 104 0.0 4
107 105 0.0 50
108 106 0.0 8
109 107 0.0 12
110 108 0.0 43
111 109 0.0 30
112 110 0.0 15
113 111 0.0 19
114 112 0.0 182
115 113 0.0 40
116 114 0.0 88
117 115 0.0 19
118 116 0.0 30
119 117 0.0 27
120 118 0.0 5
121 119 0.0 87
122 120 0.0 9
123 121 0.0 64
124 122 0.0 27
125 123 0.0 68
126 124 0.0 81
127 125 0.0 86
128 126 0.0 227
129 127 0.0 41
130 128 0.0 70
131 129 0.0 27
132 130 0.0 6
133 131 0.0 18
134 132 0.0 38
135 133 0.0 26
136 134 0.0 36
137 135 0.0 3
138 136 0.0 61
139 137 0.0 105
140 138 0.0 38
141 139 0.0 18
142 140 0.0 33
143 141 0.0 29
144 142 0.0 49
145 143 0.0 88
146 144 0.0 22
147 145 0.0 65
148 146 0.0 36
149 147 0.0 30
150 148 0.0 58
151 149 0.0 43
152 150 0.0 53
153 151 0.0 43
154 152 0.0 13
155 153 0.0 8
156 154 0.0 39
157 155 0.0 29
158 156 0.0 26
159 157 0.0 60
160 158 0.0 153
161 159 0.0 116
162 160 0.0 53
163 161 0.0 54
164 162 0.0 8
165 163 0.0 58
166 164 0.0 3
167 165 0.0 47
168 166 0.0 16
169 167 0.0 21
170 168 0.0 44
171 169 0.0 29
172 170 0.0 104
173 171 0.0 158
174 172 0.0 83
175 173 0.0 26
176 174 0.0 24
177 175 0.0 10
178 176 0.0 12
179 177 0.0 40
180 178 0.0 25
181 179 0.0 18
182 180 0.0 60
183 181 0.0 203
184 182 0.0 23
185 183 0.0 54
186 184 0.0 71
187 185 0.0 19
188 186 0.0 118
189 187 0.0 26
190 188 0.0 41
191 189 0.0 41
192 190 0.0 60
193 191 0.0 31
194 192 0.0 34
195 193 0.0 35
196 194 0.0 59
197 195 0.0 51
198 196 0.0 426
199 197 0.0 79
200 198 0.0 40
201 199 0.0 17
202 200 0.0 79
203 201 0.0 126
204 202 0.0 61
205 203 0.0 25
206 204 0.0 18
207 205 0.0 27
208 206 0.0 13
209 207 0.0 187
210 208 0.0 160
211 209 0.0 32
212 210 0.0 108
213 211 0.0 164
214 212 0.0 17
215 213 0.0 82
216 214 0.0 194
217 215 0.0 7
218 216 0.0 36
219 217 0.0 156
220 218 0.0 17
221 219 0.0 183
222 220 0.0 243
223 221 0.0 87
224 222 0.0 42
225 223 0.0 80
226 224 0.0 54
227 225 0.0 82
228 226 0.0 97
229 227 0.0 65
230 228 0.0 83
231 229 0.0 159
232 230 0.0 178
233 231 0.0 104
234 232 0.0 21
235 233 0.0 118
236 234 0.0 80
237 235 0.0 170
238 236 0.0 94
239 237 0.0 235
240 238 0.0 13
241 239 0.0 31
242 240 0.0 134
243 241 0.0 32
244 242 0.0 58
245 243 0.0 38
246 244 0.0 28
247 245 0.0 159
248 246 0.0 182
249 247 0.0 51
250 248 0.0 25
251 249 0.0 73
252 250 0.0 56
253 251 0.0 55
254 252 0.0 38
255 253 0.0 292
256 254 0.0 319
257 255 0.0 100
258 256 0.0 84
259 257 0.0 24
260 258 0.0 17
261 259 0.0 159
262 260 0.0 25
263 261 0.0 73
264 262 0.0 130
265 263 0.0 111
266 264 0.0 65
267 265 1.0 58
268 266 0.0 47
269 267 0.0 48
270 268 0.0 13
271 269 0.0 100
272 270 0.0 38
273 271 0.0 111
274 272 0.0 226
275 273 0.0 38
276 274 0.0 83
277 275 0.0 42
278 276 0.0 199
279 277 0.0 83
280 278 0.0 28
281 279 0.0 46
282 280 0.0 262
283 281 0.0 123
284 282 0.0 91
285 283 0.0 53
286 284 0.0 19
287 285 0.0 26
288 286 0.0 93
289 287 0.0 38
290 288 0.0 22
291 289 0.0 43
292 290 0.0 163
293 291 0.0 25
294 292 0.0 59
295 293 0.0 71
296 294 0.0 20
297 295 0.0 115
298 296 0.0 248
299 297 0.0 66
300 298 0.0 58
301 299 0.0 129
302 300 0.0 122
303 301 0.0 47
304 302 0.0 60
305 303 0.0 79
306 304 1.0 137
307 305 0.0 27
308 306 1.0 93
309 307 0.0 46
310 308 1.0 83
311 309 1.0 8
312 310 1.0 6
313 311 1.0 6
314 312 0.0 4
315 313 1.0 6
316 314 0.0 2
317 315 1.0 6
318 316 1.0 6
319 317 1.0 6
320 318 1.0 6
321 319 1.0 8
322 320 0.0 5
323 321 1.0 6
324 322 1.0 7
325 323 0.0 5
326 324 1.0 6
327 325 1.0 6
328 326 1.0 8
329 327 1.0 6
330 328 1.0 6
331 329 1.0 6
332 330 1.0 7
333 331 1.0 6
334 332 1.0 6
335 333 0.0 3
336 334 1.0 7
337 335 0.0 4
338 336 1.0 6
339 337 1.0 6
340 338 1.0 7
341 339 1.0 6
342 340 1.0 6
343 341 1.0 7
344 342 1.0 7
345 343 1.0 7
346 344 1.0 6
347 345 1.0 6
348 346 1.0 6
349 347 1.0 6
350 348 1.0 6
351 349 1.0 6
352 350 1.0 6
353 351 1.0 7
354 352 0.0 4
355 353 1.0 8
356 354 1.0 8
357 355 1.0 7
358 356 1.0 6
359 357 1.0 8
360 358 1.0 6
361 359 1.0 6
362 360 1.0 7
363 361 1.0 6
364 362 1.0 6
365 363 1.0 8
366 364 1.0 7
367 365 1.0 6
368 366 1.0 6
369 367 0.0 3
370 368 1.0 11
371 369 1.0 6
372 370 1.0 8
373 371 0.0 2
374 372 1.0 6
375 373 1.0 6
376 374 1.0 6
377 375 1.0 6
378 376 1.0 8
379 377 1.0 6
380 378 1.0 7
381 379 1.0 6
382 380 1.0 7
383 381 1.0 6
384 382 1.0 8
385 383 0.0 2
386 384 1.0 6
387 385 1.0 7
388 386 1.0 6
389 387 1.0 6
390 388 1.0 10
391 389 1.0 7
392 390 1.0 6
393 391 1.0 6
394 392 1.0 6
395 393 1.0 6
396 394 1.0 6
397 395 1.0 7
398 396 0.0 4
399 397 1.0 7
400 398 1.0 6
401 399 1.0 8
402 400 0.0 3
403 401 1.0 6
404 402 1.0 6
405 403 1.0 6
406 404 1.0 6
407 405 0.0 2
408 406 1.0 6
409 407 1.0 6
410 408 1.0 6
411 409 1.0 6
412 410 1.0 6
413 411 1.0 7
414 412 1.0 6
415 413 1.0 6
416 414 1.0 7
417 415 1.0 6
418 416 1.0 6
419 417 1.0 6
420 418 1.0 6
421 419 1.0 6
422 420 1.0 6
423 421 1.0 6
424 422 1.0 8
425 423 1.0 6
426 424 1.0 8
427 425 1.0 7
428 426 1.0 6
429 427 0.0 3
430 428 1.0 6
431 429 1.0 7
432 430 1.0 6
433 431 1.0 6
434 432 1.0 6
435 433 1.0 10
436 434 1.0 6
437 435 1.0 6
438 436 1.0 6
439 437 1.0 6
440 438 1.0 10
441 439 1.0 6
442 440 1.0 8
443 441 1.0 8
444 442 1.0 7
445 443 1.0 6
446 444 0.0 5
447 445 0.0 2
448 446 1.0 8
449 447 1.0 6
450 448 1.0 10
451 449 1.0 6
452 450 1.0 8
453 451 1.0 10
454 452 1.0 6
455 453 1.0 6
456 454 1.0 6
457 455 1.0 10
458 456 1.0 6
459 457 0.0 4
460 458 1.0 6
461 459 1.0 6
462 460 1.0 6
463 461 1.0 15
464 462 1.0 6
465 463 1.0 6
466 464 1.0 6
467 465 1.0 6
468 466 1.0 6
469 467 1.0 6
470 468 1.0 8
471 469 1.0 6
472 470 1.0 7
473 471 1.0 6
474 472 1.0 6
475 473 1.0 8
476 474 1.0 6
477 475 1.0 6
478 476 1.0 8
479 477 1.0 8
480 478 1.0 6
481 479 1.0 6
482 480 1.0 6
483 481 1.0 10
484 482 1.0 6
485 483 1.0 6
486 484 1.0 6
487 485 1.0 6
488 486 1.0 6
489 487 1.0 6
490 488 1.0 6
491 489 1.0 8
492 490 1.0 8
493 491 1.0 6
494 492 1.0 6
495 493 0.0 2
496 494 1.0 6
497 495 1.0 6
498 496 1.0 6
499 497 1.0 8
500 498 1.0 6
501 499 1.0 6
502 500 1.0 6
503 501 1.0 6
504 502 1.0 6
505 503 1.0 6
506 504 1.0 6
507 505 1.0 6
508 506 1.0 6
509 507 1.0 7
510 508 0.0 3
511 509 1.0 7
512 510 1.0 6
513 511 1.0 6
514 512 1.0 6
515 513 0.0 2
516 514 1.0 6
517 515 1.0 8
518 516 1.0 6
519 517 1.0 6
520 518 1.0 6
521 519 1.0 6
522 520 1.0 9
523 521 1.0 6
524 522 1.0 6
525 523 1.0 6
526 524 1.0 6
527 525 1.0 6
528 526 1.0 6
529 527 1.0 9
530 528 1.0 7
531 529 0.0 4
532 530 1.0 6
533 531 1.0 8
534 532 1.0 11
535 533 1.0 6
536 534 1.0 6
537 535 1.0 6
538 536 1.0 6
539 537 1.0 6
540 538 1.0 8
541 539 1.0 6
542 540 1.0 6
543 541 1.0 8
544 542 1.0 7
545 543 1.0 6
546 544 1.0 8
547 545 1.0 6
548 546 0.0 5
549 547 1.0 9
550 548 1.0 8
551 549 1.0 8
552 550 1.0 6
553 551 1.0 8
554 552 1.0 8
555 553 1.0 6
556 554 0.0 5
557 555 0.0 3
558 556 0.0 2
559 557 1.0 8
560 558 1.0 6
561 559 1.0 6
562 560 1.0 6
563 561 1.0 6
564 562 1.0 6
565 563 1.0 6
566 564 1.0 6
567 565 1.0 6
568 566 1.0 6
569 567 1.0 6
570 568 1.0 6
571 569 1.0 6
572 570 1.0 6
573 571 1.0 6
574 572 0.0 2
575 573 1.0 6
576 574 0.0 4
577 575 1.0 6
578 576 1.0 6
579 577 1.0 6
580 578 1.0 6
581 579 1.0 6
582 580 1.0 8
583 581 0.0 5
584 582 1.0 6
585 583 1.0 6
586 584 1.0 6
587 585 1.0 6
588 586 1.0 6
589 587 1.0 6
590 588 0.0 3
591 589 1.0 6
592 590 1.0 6
593 591 1.0 6
594 592 0.0 2
595 593 1.0 6
596 594 0.0 4
597 595 1.0 6
598 596 1.0 6
599 597 1.0 6
600 598 1.0 6
601 599 1.0 8
602 600 1.0 6
603 601 1.0 7
604 602 1.0 6
605 603 1.0 7
606 604 1.0 6
607 605 0.0 2
608 606 1.0 6
609 607 1.0 6
610 608 0.0 5
611 609 0.0 3
612 610 0.0 3
613 611 1.0 6
614 612 0.0 5
615 613 1.0 8
616 614 1.0 8
617 615 1.0 6
618 616 1.0 6
619 617 1.0 7
620 618 1.0 6
621 619 1.0 6
622 620 1.0 6
623 621 1.0 6
624 622 1.0 6
625 623 1.0 8
626 624 0.0 2
627 625 1.0 6
628 626 1.0 6
629 627 1.0 6
630 628 1.0 6
631 629 1.0 6
632 630 1.0 6
633 631 1.0 6
634 632 1.0 8
635 633 1.0 6
636 634 1.0 8
637 635 1.0 6
638 636 1.0 6
639 637 1.0 8
640 638 1.0 8
641 639 0.0 5
642 640 0.0 4
643 641 0.0 4
644 642 1.0 6
645 643 1.0 6
646 644 1.0 6
647 645 1.0 6
648 646 1.0 8
649 647 1.0 6
650 648 0.0 4
651 649 1.0 6
652 650 1.0 8
653 651 1.0 6
654 652 1.0 6
655 653 1.0 6
656 654 1.0 6
657 655 1.0 6
658 656 1.0 6
659 657 1.0 6
660 658 1.0 8
661 659 1.0 8
662 660 1.0 6
663 661 1.0 8
664 662 1.0 9
665 663 1.0 6
666 664 1.0 6
667 665 1.0 6
668 666 1.0 6
669 667 1.0 10
670 668 1.0 6
671 669 1.0 6
672 670 1.0 6
673 671 1.0 11
674 672 1.0 10
675 673 1.0 8
676 674 1.0 6
677 675 1.0 6
678 676 1.0 6
679 677 0.0 5
680 678 1.0 6
681 679 0.0 2
682 680 1.0 9
683 681 1.0 6
684 682 1.0 8
685 683 1.0 7
686 684 1.0 6
687 685 1.0 6
688 686 1.0 7
689 687 0.0 3
690 688 1.0 7
691 689 0.0 2
692 690 1.0 6
693 691 1.0 6
694 692 1.0 8
695 693 1.0 8
696 694 1.0 6
697 695 1.0 6
698 696 0.0 2
699 697 1.0 8
700 698 1.0 6
701 699 1.0 8
702 700 1.0 6
703 701 1.0 6
704 702 1.0 9
705 703 1.0 6
706 704 1.0 8
707 705 1.0 11
708 706 1.0 6
709 707 1.0 6
710 708 1.0 6
711 709 1.0 6
712 710 1.0 8
713 711 1.0 6
714 712 1.0 6
715 713 1.0 6
716 714 0.0 5
717 715 1.0 6
718 716 1.0 6
719 717 1.0 6
720 718 1.0 6
721 719 1.0 6
722 720 1.0 7
723 721 1.0 6
724 722 1.0 6
725 723 1.0 6
726 724 1.0 6
727 725 1.0 10
728 726 1.0 6
729 727 1.0 6
730 728 1.0 6
731 729 1.0 6
732 730 1.0 6
733 731 1.0 7
734 732 1.0 6
735 733 1.0 8
736 734 1.0 7
737 735 1.0 6
738 736 1.0 6
739 737 1.0 14
740 738 1.0 6
741 739 1.0 6
742 740 1.0 12
743 741 1.0 6
744 742 1.0 6
745 743 1.0 6
746 744 1.0 6
747 745 1.0 6
748 746 1.0 6
749 747 0.0 3
750 748 1.0 6
751 749 1.0 6
752 750 1.0 6
753 751 1.0 7
754 752 1.0 6
755 753 1.0 6
756 754 1.0 6
757 755 1.0 8
758 756 0.0 2
759 757 1.0 6
760 758 1.0 6
761 759 1.0 6
762 760 1.0 6
763 761 1.0 6
764 762 1.0 6
765 763 1.0 6
766 764 1.0 6
767 765 1.0 6
768 766 0.0 4
769 767 1.0 8
770 768 1.0 6
771 769 0.0 2
772 770 1.0 10
773 771 1.0 8
774 772 1.0 6
775 773 1.0 6
776 774 1.0 6
777 775 0.0 3
778 776 1.0 6
779 777 1.0 6
780 778 0.0 6
781 779 1.0 8
782 780 1.0 6
783 781 1.0 9
784 782 1.0 6
785 783 1.0 6
786 784 1.0 8
787 785 1.0 8
788 786 1.0 6
789 787 0.0 5
790 788 1.0 6
791 789 1.0 6
792 790 1.0 6
793 791 1.0 6
794 792 1.0 6
795 793 1.0 6
796 794 1.0 8
797 795 1.0 6
798 796 0.0 2
799 797 1.0 8
800 798 1.0 7
801 799 1.0 6