222 lines
8.7 KiB
Python
222 lines
8.7 KiB
Python
#!/usr/bin/env python
|
|
# coding=utf-8
|
|
'''
|
|
Author: JiangJi
|
|
Email: johnjim0816@gmail.com
|
|
Date: 2021-04-29 12:53:54
|
|
LastEditor: JiangJi
|
|
LastEditTime: 2021-12-22 15:41:19
|
|
Discription:
|
|
Environment:
|
|
'''
|
|
import copy
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.optim as optim
|
|
import torch.nn.functional as F
|
|
from torch.distributions import Normal
|
|
import numpy as np
|
|
import random
|
|
device=torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
class ReplayBuffer:
|
|
def __init__(self, capacity):
|
|
self.capacity = capacity # 经验回放的容量
|
|
self.buffer = [] # 缓冲区
|
|
self.position = 0
|
|
|
|
def push(self, state, action, reward, next_state, done):
|
|
''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition)
|
|
'''
|
|
if len(self.buffer) < self.capacity:
|
|
self.buffer.append(None)
|
|
self.buffer[self.position] = (state, action, reward, next_state, done)
|
|
self.position = (self.position + 1) % self.capacity
|
|
|
|
def sample(self, batch_size):
|
|
batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移
|
|
state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等
|
|
return state, action, reward, next_state, done
|
|
|
|
def __len__(self):
|
|
''' 返回当前存储的量
|
|
'''
|
|
return len(self.buffer)
|
|
|
|
class ValueNet(nn.Module):
|
|
def __init__(self, state_dim, hidden_dim, init_w=3e-3):
|
|
super(ValueNet, self).__init__()
|
|
|
|
self.linear1 = nn.Linear(state_dim, hidden_dim)
|
|
self.linear2 = nn.Linear(hidden_dim, hidden_dim)
|
|
self.linear3 = nn.Linear(hidden_dim, 1)
|
|
|
|
self.linear3.weight.data.uniform_(-init_w, init_w)
|
|
self.linear3.bias.data.uniform_(-init_w, init_w)
|
|
|
|
def forward(self, state):
|
|
x = F.relu(self.linear1(state))
|
|
x = F.relu(self.linear2(x))
|
|
x = self.linear3(x)
|
|
return x
|
|
|
|
|
|
class SoftQNet(nn.Module):
|
|
def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3):
|
|
super(SoftQNet, self).__init__()
|
|
|
|
self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim)
|
|
self.linear2 = nn.Linear(hidden_dim, hidden_dim)
|
|
self.linear3 = nn.Linear(hidden_dim, 1)
|
|
|
|
self.linear3.weight.data.uniform_(-init_w, init_w)
|
|
self.linear3.bias.data.uniform_(-init_w, init_w)
|
|
|
|
def forward(self, state, action):
|
|
x = torch.cat([state, action], 1)
|
|
x = F.relu(self.linear1(x))
|
|
x = F.relu(self.linear2(x))
|
|
x = self.linear3(x)
|
|
return x
|
|
|
|
|
|
class PolicyNet(nn.Module):
|
|
def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2):
|
|
super(PolicyNet, self).__init__()
|
|
|
|
self.log_std_min = log_std_min
|
|
self.log_std_max = log_std_max
|
|
|
|
self.linear1 = nn.Linear(state_dim, hidden_dim)
|
|
self.linear2 = nn.Linear(hidden_dim, hidden_dim)
|
|
|
|
self.mean_linear = nn.Linear(hidden_dim, action_dim)
|
|
self.mean_linear.weight.data.uniform_(-init_w, init_w)
|
|
self.mean_linear.bias.data.uniform_(-init_w, init_w)
|
|
|
|
self.log_std_linear = nn.Linear(hidden_dim, action_dim)
|
|
self.log_std_linear.weight.data.uniform_(-init_w, init_w)
|
|
self.log_std_linear.bias.data.uniform_(-init_w, init_w)
|
|
|
|
def forward(self, state):
|
|
x = F.relu(self.linear1(state))
|
|
x = F.relu(self.linear2(x))
|
|
|
|
mean = self.mean_linear(x)
|
|
log_std = self.log_std_linear(x)
|
|
log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)
|
|
|
|
return mean, log_std
|
|
|
|
def evaluate(self, state, epsilon=1e-6):
|
|
mean, log_std = self.forward(state)
|
|
std = log_std.exp()
|
|
|
|
normal = Normal(mean, std)
|
|
z = normal.sample()
|
|
action = torch.tanh(z)
|
|
|
|
log_prob = normal.log_prob(z) - torch.log(1 - action.pow(2) + epsilon)
|
|
log_prob = log_prob.sum(-1, keepdim=True)
|
|
|
|
return action, log_prob, z, mean, log_std
|
|
|
|
|
|
def get_action(self, state):
|
|
state = torch.FloatTensor(state).unsqueeze(0).to(device)
|
|
mean, log_std = self.forward(state)
|
|
std = log_std.exp()
|
|
|
|
normal = Normal(mean, std)
|
|
z = normal.sample()
|
|
action = torch.tanh(z)
|
|
|
|
action = action.detach().cpu().numpy()
|
|
return action[0]
|
|
|
|
class SAC:
|
|
def __init__(self,state_dim,action_dim,cfg) -> None:
|
|
self.batch_size = cfg.batch_size
|
|
self.memory = ReplayBuffer(cfg.capacity)
|
|
self.device = cfg.device
|
|
self.value_net = ValueNet(state_dim, cfg.hidden_dim).to(self.device)
|
|
self.target_value_net = ValueNet(state_dim, cfg.hidden_dim).to(self.device)
|
|
self.soft_q_net = SoftQNet(state_dim, action_dim, cfg.hidden_dim).to(self.device)
|
|
self.policy_net = PolicyNet(state_dim, action_dim, cfg.hidden_dim).to(self.device)
|
|
self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=cfg.value_lr)
|
|
self.soft_q_optimizer = optim.Adam(self.soft_q_net.parameters(), lr=cfg.soft_q_lr)
|
|
self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.policy_lr)
|
|
for target_param, param in zip(self.target_value_net.parameters(), self.value_net.parameters()):
|
|
target_param.data.copy_(param.data)
|
|
self.value_criterion = nn.MSELoss()
|
|
self.soft_q_criterion = nn.MSELoss()
|
|
def update(self, gamma=0.99,mean_lambda=1e-3,
|
|
std_lambda=1e-3,
|
|
z_lambda=0.0,
|
|
soft_tau=1e-2,
|
|
):
|
|
if len(self.memory) < self.batch_size:
|
|
return
|
|
state, action, reward, next_state, done = self.memory.sample(self.batch_size)
|
|
state = torch.FloatTensor(state).to(self.device)
|
|
next_state = torch.FloatTensor(next_state).to(self.device)
|
|
action = torch.FloatTensor(action).to(self.device)
|
|
reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device)
|
|
done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device)
|
|
expected_q_value = self.soft_q_net(state, action)
|
|
expected_value = self.value_net(state)
|
|
new_action, log_prob, z, mean, log_std = self.policy_net.evaluate(state)
|
|
|
|
|
|
target_value = self.target_value_net(next_state)
|
|
next_q_value = reward + (1 - done) * gamma * target_value
|
|
q_value_loss = self.soft_q_criterion(expected_q_value, next_q_value.detach())
|
|
|
|
expected_new_q_value = self.soft_q_net(state, new_action)
|
|
next_value = expected_new_q_value - log_prob
|
|
value_loss = self.value_criterion(expected_value, next_value.detach())
|
|
|
|
log_prob_target = expected_new_q_value - expected_value
|
|
policy_loss = (log_prob * (log_prob - log_prob_target).detach()).mean()
|
|
|
|
|
|
mean_loss = mean_lambda * mean.pow(2).mean()
|
|
std_loss = std_lambda * log_std.pow(2).mean()
|
|
z_loss = z_lambda * z.pow(2).sum(1).mean()
|
|
|
|
policy_loss += mean_loss + std_loss + z_loss
|
|
|
|
self.soft_q_optimizer.zero_grad()
|
|
q_value_loss.backward()
|
|
self.soft_q_optimizer.step()
|
|
|
|
self.value_optimizer.zero_grad()
|
|
value_loss.backward()
|
|
self.value_optimizer.step()
|
|
|
|
self.policy_optimizer.zero_grad()
|
|
policy_loss.backward()
|
|
self.policy_optimizer.step()
|
|
|
|
for target_param, param in zip(self.target_value_net.parameters(), self.value_net.parameters()):
|
|
target_param.data.copy_(
|
|
target_param.data * (1.0 - soft_tau) + param.data * soft_tau
|
|
)
|
|
def save(self, path):
|
|
torch.save(self.value_net.state_dict(), path + "sac_value")
|
|
torch.save(self.value_optimizer.state_dict(), path + "sac_value_optimizer")
|
|
torch.save(self.soft_q_net.state_dict(), path + "sac_soft_q")
|
|
torch.save(self.soft_q_optimizer.state_dict(), path + "sac_soft_q_optimizer")
|
|
|
|
torch.save(self.policy_net.state_dict(), path + "sac_policy")
|
|
torch.save(self.policy_optimizer.state_dict(), path + "sac_policy_optimizer")
|
|
|
|
def load(self, path):
|
|
self.value_net.load_state_dict(torch.load(path + "sac_value"))
|
|
self.value_optimizer.load_state_dict(torch.load(path + "sac_value_optimizer"))
|
|
self.target_value_net = copy.deepcopy(self.value_net)
|
|
|
|
self.soft_q_net.load_state_dict(torch.load(path + "sac_soft_q"))
|
|
self.soft_q_optimizer.load_state_dict(torch.load(path + "sac_soft_q_optimizer"))
|
|
|
|
self.policy_net.load_state_dict(torch.load(path + "sac_policy"))
|
|
self.policy_optimizer.load_state_dict(torch.load(path + "sac_policy_optimizer")) |