add some codes

codes/ddpg/ddpg.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-09 20:25:52
+@LastEditor: John
+@LastEditTime: 2020-06-14 11:43:17
+@Discription: 
+@Environment: python 3.7.7
+'''
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+
+from model import Actor, Critic
+from memory import ReplayBuffer
+
+
+class DDPG:
+    def __init__(self, n_states, n_actions, hidden_dim=30, device="cpu", critic_lr=1e-3,
+                 actor_lr=1e-4, gamma=0.99, soft_tau=1e-2, memory_capacity=100000, batch_size=128):
+        self.device = device
+        self.critic = Critic(n_states, n_actions, hidden_dim).to(device)
+        self.actor = Actor(n_states, n_actions, hidden_dim).to(device)
+        self.target_critic = Critic(n_states, n_actions, hidden_dim).to(device)
+        self.target_actor = Actor(n_states, n_actions, hidden_dim).to(device)
+
+        for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()):
+            target_param.data.copy_(param.data)
+        for target_param, param in zip(self.target_actor.parameters(), self.actor.parameters()):
+            target_param.data.copy_(param.data)
+
+        self.critic_optimizer = optim.Adam(
+            self.critic.parameters(),  lr=critic_lr)
+        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr)
+        self.critic_criterion = nn.MSELoss()
+        self.memory = ReplayBuffer(memory_capacity)
+        self.batch_size = batch_size
+        self.soft_tau = soft_tau
+        self.gamma = gamma
+
+    def select_action(self, state):
+        return self.actor.select_action(state)
+
+    def update(self):
+        if len(self.memory) < self.batch_size:
+            return
+        state, action, reward, next_state, done = self.memory.sample(
+            self.batch_size) 
+        # 将所有变量转为张量
+        state = torch.FloatTensor(state).to(self.device)
+        next_state = torch.FloatTensor(next_state).to(self.device)
+        action = torch.FloatTensor(action).to(self.device)
+        reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device)
+        
+        done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device)
+        policy_loss = self.critic(state, self.actor(state))
+        policy_loss = -policy_loss.mean()
+
+        next_action = self.target_actor(next_state)
+        target_value = self.target_critic(next_state, next_action.detach())
+        expected_value = reward + (1.0 - done) * self.gamma * target_value
+        expected_value = torch.clamp(expected_value, -np.inf, np.inf)
+        
+        value = self.critic(state, action)
+
+        value_loss = self.critic_criterion(value, expected_value.detach())
+
+        self.actor_optimizer.zero_grad()
+        policy_loss.backward()
+        self.actor_optimizer.step()
+
+        self.critic_optimizer.zero_grad()
+        value_loss.backward()
+        self.critic_optimizer.step()
+        for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()):
+            target_param.data.copy_(
+                target_param.data * (1.0 - self.soft_tau) +
+                param.data * self.soft_tau
+            )
+        for target_param, param in zip(self.target_actor.parameters(), self.actor.parameters()):
+            target_param.data.copy_(
+                target_param.data * (1.0 - self.soft_tau) +
+                param.data * self.soft_tau
+            )

codes/ddpg/env.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-10 15:28:30
+@LastEditor: John
+@LastEditTime: 2020-06-12 22:49:18
+@Discription: 
+@Environment: python 3.7.7
+'''
+import gym
+import numpy as np
+
+class NormalizedActions(gym.ActionWrapper):
+
+    def action(self, action):
+        
+        low_bound   = self.action_space.low
+        upper_bound = self.action_space.high
+        action = low_bound + (action + 1.0) * 0.5 * (upper_bound - low_bound)
+        action = np.clip(action, low_bound, upper_bound)
+        
+        return action
+
+    def reverse_action(self, action):
+        low_bound   = self.action_space.low
+        upper_bound = self.action_space.high
+        action = 2 * (action - low_bound) / (upper_bound - low_bound) - 1
+        action = np.clip(action, low_bound, upper_bound)
+        return action

codes/ddpg/main.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 20:58:21
+@LastEditor: John
+@LastEditTime: 2020-07-20 23:01:02
+@Discription: 
+@Environment: python 3.7.7
+'''
+import torch
+import gym
+
+from ddpg import DDPG
+from env import NormalizedActions
+from noise import OUNoise
+from plot import plot
+
+import argparse
+
+def get_args():
+    '''模型建立好之后只需要在这里调参
+    '''
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--gamma", default=0.99, type=float) # q-learning中的gamma
+    parser.add_argument("--critic_lr", default=1e-3, type=float) # critic学习率
+    parser.add_argument("--actor_lr", default=1e-4, type=float)
+
+    parser.add_argument("--memory_capacity", default=10000, type=int,help="capacity of Replay Memory")
+
+    parser.add_argument("--batch_size", default=128, type=int,help="batch size of memory sampling")
+    parser.add_argument("--max_episodes", default=200, type=int)
+    parser.add_argument("--max_steps", default=200, type=int)
+    parser.add_argument("--target_update", default=4, type=int,help="when(every default 10 eisodes) to update target net ")
+    config = parser.parse_args()
+
+    return config
+
+if __name__ == "__main__":
+
+    cfg = get_args()
+    env = NormalizedActions(gym.make("Pendulum-v0"))
+    
+    # 增加action噪声
+    ou_noise = OUNoise(env.action_space)
+    
+    n_states  = env.observation_space.shape[0] 
+    n_actions = env.action_space.shape[0] 
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    agent=DDPG(n_states,n_actions,device="cpu", critic_lr=1e-3,
+                 actor_lr=1e-4, gamma=0.99, soft_tau=1e-2, memory_capacity=100000, batch_size=128)
+
+    rewards  = []
+    moving_average_rewards = []
+    for i_episode in range(1,cfg.max_episodes+1):
+        state=env.reset()
+        ou_noise.reset()
+        ep_reward = 0
+        for i_step in range(1,cfg.max_steps+1):
+            action = agent.select_action(state)   
+            action = ou_noise.get_action(action, i_step) # 即paper中的random process
+            next_state, reward, done, _ = env.step(action)
+            ep_reward += reward
+            agent.memory.push(state, action, reward, next_state, done)
+            agent.update()
+            state = next_state
+            if done:
+                break
+        print('Episode:', i_episode, ' Reward: %i' % int(ep_reward),)
+        rewards.append(ep_reward)
+        #
+        if i_episode == 1:
+            moving_average_rewards.append(ep_reward)
+        else:
+            moving_average_rewards.append(
+                0.9*moving_average_rewards[-1]+0.1*ep_reward)
+    print('Complete！')
+    import os
+    import numpy as np
+    output_path = os.path.dirname(__file__)+"/result/"
+    if not os.path.exists(output_path):
+        os.mkdir(output_path)
+    np.save(output_path+"rewards.npy", rewards)
+    np.save(output_path+"moving_average_rewards.npy", moving_average_rewards)
+  
+    plot(rewards)
+    plot(moving_average_rewards,ylabel="moving_average_rewards")

codes/ddpg/memory.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-10 15:27:16
+@LastEditor: John
+@LastEditTime: 2020-06-13 00:29:45
+@Discription: 
+@Environment: python 3.7.7
+'''
+import random
+import numpy as np
+
+class ReplayBuffer:
+    
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self.buffer = []
+        self.position = 0
+    
+    def push(self, state, action, reward, next_state, done):
+        if len(self.buffer) < self.capacity:
+            self.buffer.append(None)
+        self.buffer[self.position] = (state, action, reward, next_state, done)
+        self.position = (self.position + 1) % self.capacity
+    
+    def sample(self, batch_size):
+        batch = random.sample(self.buffer, batch_size)
+        state_batch, action_batch, reward_batch, next_state_batch, done_batch = map(np.stack, zip(*batch))
+        return state_batch, action_batch, reward_batch, next_state_batch, done_batch
+    
+    def __len__(self):
+        return len(self.buffer)

codes/ddpg/model.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-10 15:03:59
+@LastEditor: John
+@LastEditTime: 2020-06-14 11:42:45
+@Discription: 
+@Environment: python 3.7.7
+'''
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class Critic(nn.Module):
+    def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3):
+        super(Critic, self).__init__()
+        
+        self.linear1 = nn.Linear(n_obs + n_actions, hidden_size)
+        self.linear2 = nn.Linear(hidden_size, hidden_size)
+        self.linear3 = nn.Linear(hidden_size, 1)
+    
+        self.linear3.weight.data.uniform_(-init_w, init_w)
+        self.linear3.bias.data.uniform_(-init_w, init_w)
+        
+    def forward(self, state, action):
+        x = torch.cat([state, action], 1)
+        x = F.relu(self.linear1(x))
+        x = F.relu(self.linear2(x))
+        x = self.linear3(x)
+        return x
+
+class Actor(nn.Module):
+    def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3):
+        super(Actor, self).__init__()  
+        self.linear1 = nn.Linear(n_obs, hidden_size)
+        self.linear2 = nn.Linear(hidden_size, hidden_size)
+        self.linear3 = nn.Linear(hidden_size, n_actions)
+        
+        self.linear3.weight.data.uniform_(-init_w, init_w)
+        self.linear3.bias.data.uniform_(-init_w, init_w)
+        
+    def forward(self, x):
+        x = F.relu(self.linear1(x))
+        x = F.relu(self.linear2(x))
+        x = F.tanh(self.linear3(x))
+        return x
+    
+    def select_action(self, state):
+        
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        state  = torch.FloatTensor(state).unsqueeze(0).to(device)
+        # print(state)
+        action = self.forward(state)
+        return action.detach().cpu().numpy()[0, 0]

codes/ddpg/noise.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 20:58:59
+@LastEditor: John
+@LastEditTime: 2020-06-11 20:59:20
+@Discription: 
+@Environment: python 3.7.7
+'''
+import numpy as np
+
+class OUNoise(object):
+    def __init__(self, action_space, mu=0.0, theta=0.15, max_sigma=0.3, min_sigma=0.3, decay_period=100000):
+        self.mu           = mu
+        self.theta        = theta
+        self.sigma        = max_sigma
+        self.max_sigma    = max_sigma
+        self.min_sigma    = min_sigma
+        self.decay_period = decay_period
+        self.n_actions   = action_space.shape[0]
+        self.low          = action_space.low
+        self.high         = action_space.high
+        self.reset()
+        
+    def reset(self):
+        self.obs = np.ones(self.n_actions) * self.mu
+        
+    def evolve_obs(self):
+        x  = self.obs
+        dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.n_actions)
+        self.obs = x + dx
+        return self.obs
+    
+    def get_action(self, action, t=0):
+        ou_obs = self.evolve_obs()
+        self.sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(1.0, t / self.decay_period)
+        return np.clip(action + ou_obs, self.low, self.high)

codes/ddpg/plot.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 16:30:09
+@LastEditor: John
+@LastEditTime: 2020-06-12 11:34:52
+@Discription: 
+@Environment: python 3.7.7
+'''
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sns; sns.set()
+import numpy as np
+import os 
+
+# def plot(item,ylabel='rewards'):
+#     plt.figure()
+#     plt.plot(np.arange(len(item)), item)
+#     plt.title(ylabel+' of DDPG') 
+#     plt.ylabel(ylabel)
+#     plt.xlabel('episodes')
+#     plt.savefig(os.path.dirname(__file__)+"/result/"+ylabel+".png")
+#     plt.show()
+
+def plot(item,ylabel='rewards'):
+    df = pd.DataFrame(dict(time=np.arange(500),
+                       value=np.random.randn(500).cumsum()))
+    g = sns.relplot(x="time", y="value", kind="line", data=df)
+    g.fig.autofmt_xdate()
+    # time = range(len(item))
+    # sns.set(style="darkgrid", font_scale=1.5)
+    # sns.lineplot(time=time, data=item, color="r", condition="behavior_cloning")
+    # # sns.tsplot(time=time, data=x2, color="b", condition="dagger")
+    # plt.ylabel("Reward")
+    # plt.xlabel("Iteration Number")
+    # plt.title("Imitation Learning")
+
+    plt.show()
+if __name__ == "__main__":
+
+    output_path = os.path.dirname(__file__)+"/result/"
+    rewards=np.load(output_path+"rewards.npy", )
+    moving_average_rewards=np.load(output_path+"moving_average_rewards.npy",)
+    plot(rewards)
+    plot(moving_average_rewards,ylabel='moving_average_rewards')