bacow/easy-rl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新算法模版

Browse Source
This commit is contained in:
johnjim0816
2022-11-06 12:15:36 +08:00
parent 466a17707f
commit dc78698262
256 changed files with 17282 additions and 10229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
projects/codes/MonteCarlo/agent.py
View File

@@ -5,7 +5,7 @@ Author: John
Email: johnjim0816@gmail.com
Date: 2021-03-12 16:14:34
LastEditor: John
LastEditTime: 2022-08-15 18:10:13
LastEditTime: 2022-11-06 01:04:57
Discription:
Environment:
'''
@@ -17,15 +17,16 @@ import dill
class FisrtVisitMC:
''' On-Policy First-Visit MC Control
'''
def __init__(self,n_actions,cfg):
self.n_actions = n_actions
def __init__(self,cfg):
self.n_actions = cfg.n_actions
self.epsilon = cfg.epsilon
self.gamma = cfg.gamma
self.Q_table = defaultdict(lambda: np.zeros(n_actions))
self.Q_table = defaultdict(lambda: np.zeros(cfg.n_actions))
self.returns_sum = defaultdict(float) # 保存return之和
self.returns_count = defaultdict(float)
def sample(self,state):
def sample_action(self,state):
state = str(state)
if state in self.Q_table.keys():
best_action = np.argmax(self.Q_table[state])
action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions
@@ -34,7 +35,8 @@ class FisrtVisitMC:
else:
action = np.random.randint(0,self.n_actions)
return action
def predict(self,state):
def predict_action(self,state):
state = str(state)
if state in self.Q_table.keys():
best_action = np.argmax(self.Q_table[state])
action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions
@@ -46,19 +48,20 @@ class FisrtVisitMC:
def update(self,one_ep_transition):
# Find all (state, action) pairs we've visited in this one_ep_transition
# We convert each state to a tuple so that we can use it as a dict key
sa_in_episode = set([(tuple(x[0]), x[1]) for x in one_ep_transition])
sa_in_episode = set([(str(x[0]), x[1]) for x in one_ep_transition])
for state, action in sa_in_episode:
sa_pair = (state, action)
# Find the first occurence of the (state, action) pair in the one_ep_transition
first_occurence_idx = next(i for i,x in enumerate(one_ep_transition)
if x[0] == state and x[1] == action)
if str(x[0]) == state and x[1] == action)
# Sum up all rewards since the first occurance
G = sum([x[2]*(self.gamma**i) for i,x in enumerate(one_ep_transition[first_occurence_idx:])])
# Calculate average return for this state over all sampled episodes
self.returns_sum[sa_pair] += G
self.returns_count[sa_pair] += 1.0
self.Q_table[state][action] = self.returns_sum[sa_pair] / self.returns_count[sa_pair]
def save(self,path=None):
def save_model(self,path=None):
'''把 Q表格 的数据保存到文件中
'''
from pathlib import Path
@@ -69,7 +72,7 @@ class FisrtVisitMC:
pickle_module=dill
)
def load(self, path=None):
def load_model(self, path=None):
'''从文件中读取数据到 Q表格
'''
self.Q_table =torch.load(f=path+"Q_table",pickle_module=dill)