add some codes

codes/dqn/dqn.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-12 00:50:49
+@LastEditor: John
+@LastEditTime: 2020-06-14 13:56:45
+@Discription: 
+@Environment: python 3.7.7
+'''
+'''off-policy
+'''
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+import random
+import math
+import numpy as np
+from memory import ReplayBuffer
+from model import FCN
+
+class DQN:
+    def __init__(self, n_states, n_actions, gamma=0.99, epsilon_start=0.9, epsilon_end=0.05, epsilon_decay=200, memory_capacity=10000, policy_lr=0.01,batch_size=128, device="cpu"):
+        self.actions_count = 0
+        self.n_actions = n_actions
+        self.device = device
+        self.gamma = gamma
+        self.epsilon = 0
+        self.epsilon_start = epsilon_start
+        self.epsilon_end = epsilon_end
+        self.epsilon_decay = epsilon_decay
+        self.batch_size = batch_size
+        self.policy_net = FCN(n_states,n_actions).to(self.device)
+        self.target_net = FCN(n_states,n_actions).to(self.device)
+        self.target_net.load_state_dict(self.policy_net.state_dict())
+        self.target_net.eval()  # 不启用 BatchNormalization 和 Dropout
+        self.optimizer = optim.Adam(self.policy_net.parameters(),lr=policy_lr)
+        self.loss = 0
+        self.memory = ReplayBuffer(memory_capacity)
+
+    def select_action(self,state):
+        '''选择工作
+        Args:
+            state [array]: 状态
+        Returns:
+            [array]: 动作
+        '''
+        self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
+            math.exp(-1. * self.actions_count / self.epsilon_decay)
+        self.actions_count += 1
+        if random.random() > self.epsilon:
+            with torch.no_grad():
+                state   = torch.tensor([state],device=self.device,dtype=torch.float32) # 先转为张量便于丢给神经网络,state元素数据原本为float64；注意state=torch.tensor(state).unsqueeze(0)跟state=torch.tensor([state])等价
+                q_value = self.policy_net(state) # tensor([[-0.0798, -0.0079]], grad_fn=<AddmmBackward>)
+                action  = q_value.max(1)[1].item()
+        else:
+            action = random.randrange(self.n_actions)
+        return action
+    def update(self):
+
+        if len(self.memory) < self.batch_size:
+            return
+
+        state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample(self.batch_size)
+        
+        state_batch = torch.tensor(state_batch,device=self.device,dtype=torch.float) # 例如tensor([[-4.5543e-02, -2.3910e-01,  1.8344e-02,  2.3158e-01],...,[-1.8615e-02, -2.3921e-01, -1.1791e-02,  2.3400e-01]])
+        action_batch  = torch.tensor(action_batch,device=self.device).unsqueeze(1) # 例如tensor([[1],...,[0]])
+        reward_batch  = torch.tensor(reward_batch,device=self.device,dtype=torch.float) # tensor([1., 1.,...,1])
+        next_state_batch = torch.tensor(next_state_batch,device=self.device,dtype=torch.float)
+        done_batch = torch.tensor(np.float32(done_batch),device=self.device).unsqueeze(1) # 将bool转为float然后转为张量
+        # Compute Q(s_t, a) - the model computes Q(s_t), then we select the
+        # columns of actions taken. These are the actions which would've been taken
+        # for each batch state according to policy_net
+        q_values = self.policy_net(state_batch).gather(1, action_batch) # 等价于self.forward       
+        # Compute V(s_{t+1}) for all next states.
+        # Expected values of actions for non_final_next_states are computed based
+        # on the "older" target_net; selecting their best reward with max(1)[0].
+        # This is merged based on the mask, such that we'll have either the expected
+        # state value or 0 in case the state was final.
+     
+        next_state_values = self.target_net(
+            next_state_batch).max(1)[0].detach() # tensor([ 0.0060, -0.0171,...,])
+        # Compute the expected Q values
+        expected_q_values = reward_batch + self.gamma * next_state_values * (1-done_batch[0])
+
+        # Compute Huber loss
+        # self.loss = nn.MSELoss(q_values, expected_q_values.unsqueeze(1)) 
+        self.loss = nn.MSELoss()(q_values,expected_q_values.unsqueeze(1))
+        # Optimize the model
+        self.optimizer.zero_grad() # zero_grad clears old gradients from the last step (otherwise you’d just accumulate the gradients from all loss.backward() calls).
+        self.loss.backward() # loss.backward() computes the derivative of the loss w.r.t. the parameters (or anything requiring gradients) using backpropagation.
+        for param in self.policy_net.parameters(): # clip防止梯度爆炸
+            param.grad.data.clamp_(-1, 1)
+        self.optimizer.step() # causes the optimizer to take a step based on the gradients of the parameters.
+        
+

codes/dqn/main.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-12 00:48:57
+@LastEditor: John
+@LastEditTime: 2020-07-20 23:02:16
+@Discription: 
+@Environment: python 3.7.7
+'''
+'''未完成
+'''
+import gym
+import torch
+from dqn import DQN
+from plot import plot
+import argparse
+def get_args():
+    '''模型建立好之后只需要在这里调参
+    '''
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--gamma", default=0.99,
+                        type=float)  # q-learning中的gamma
+    parser.add_argument("--epsilon_start", default=0.95,
+                        type=float)  # 基于贪心选择action对应的参数epsilon
+    parser.add_argument("--epsilon_end", default=0.01, type=float)
+    parser.add_argument("--epsilon_decay", default=200, type=float)
+    parser.add_argument("--policy_lr", default=0.01, type=float)
+    parser.add_argument("--memory_capacity", default=1000,
+                        type=int, help="capacity of Replay Memory")
+
+    parser.add_argument("--batch_size", default=32, type=int,
+                        help="batch size of memory sampling")
+    parser.add_argument("--max_episodes", default=200, type=int)
+    parser.add_argument("--max_steps", default=200, type=int)
+    parser.add_argument("--target_update", default=2, type=int,
+                        help="when(every default 10 eisodes) to update target net ")
+    config = parser.parse_args()
+
+    return config
+
+
+if __name__ == "__main__":
+
+    cfg = get_args()
+    # if gpu is to be used
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    env = gym.make('CartPole-v0').unwrapped
+    env.seed(1)
+    n_states = env.observation_space.shape[0]
+    n_actions = env.action_space.n
+    agent = DQN(n_states=n_states, n_actions=n_actions, device=device, gamma=cfg.gamma, epsilon_start=cfg.epsilon_start,
+                epsilon_end=cfg.epsilon_end, epsilon_decay=cfg.epsilon_decay,policy_lr=cfg.policy_lr, memory_capacity=cfg.memory_capacity, batch_size=cfg.batch_size)
+    rewards = []
+    moving_average_rewards = []
+    for i_episode in range(1, cfg.max_episodes+1):
+        # Initialize the environment and state
+        state = env.reset()
+        ep_reward = 0
+        for t in range(1, cfg.max_steps+1):
+            # Select and perform an action
+            action = agent.select_action(state)
+            next_state, reward, done, _ = env.step(action)
+            ep_reward += reward
+            # Store the transition in memory
+            agent.memory.push(state,action,reward,next_state,done)
+            # Move to the next state
+            state = next_state
+            # Perform one step of the optimization (on the target network)
+            agent.update()
+            if done:
+                break
+
+        # Update the target network, copying all weights and biases in DQN
+        if i_episode % cfg.target_update == 0:
+            agent.target_net.load_state_dict(agent.policy_net.state_dict())
+        print('Episode:', i_episode, ' Reward: %i' %
+              int(ep_reward), 'Explore: %.2f' % agent.epsilon)
+        rewards.append(ep_reward)
+        # 计算滑动窗口的reward
+        if i_episode == 1:
+            moving_average_rewards.append(ep_reward)
+        else:
+            moving_average_rewards.append(
+                0.9*moving_average_rewards[-1]+0.1*ep_reward)
+
+    import os
+    import numpy as np
+    output_path = os.path.dirname(__file__)+"/result/"
+    if not os.path.exists(output_path):
+        os.mkdir(output_path)
+    np.save(output_path+"rewards.npy", rewards)
+    np.save(output_path+"moving_average_rewards.npy", moving_average_rewards)
+    print('Complete！')
+    plot(rewards)
+    plot(moving_average_rewards, ylabel="moving_average_rewards")

codes/dqn/memory.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-10 15:27:16
+@LastEditor: John
+@LastEditTime: 2020-06-14 11:36:24
+@Discription: 
+@Environment: python 3.7.7
+'''
+import random
+import numpy as np
+
+class ReplayBuffer:
+    
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self.buffer = []
+        self.position = 0
+    
+    def push(self, state, action, reward, next_state, done):
+        if len(self.buffer) < self.capacity:
+            self.buffer.append(None)
+        self.buffer[self.position] = (state, action, reward, next_state, done)
+        self.position = (self.position + 1) % self.capacity
+    
+    def sample(self, batch_size):
+        batch = random.sample(self.buffer, batch_size)
+        state, action, reward, next_state, done =  zip(*batch)
+        return state, action, reward, next_state, done
+    
+    def __len__(self):
+        return len(self.buffer)
+

codes/dqn/model.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-12 00:47:02
+@LastEditor: John
+@LastEditTime: 2020-06-14 11:23:04
+@Discription: 
+@Environment: python 3.7.7
+'''
+import torch.nn as nn
+import torch.nn.functional as F
+
+class FCN(nn.Module):
+    def __init__(self, n_states=4, n_actions=18):
+        """
+        Initialize a deep Q-learning network for testing algorithm
+            n_states: number of features of input.
+            n_actions: number of action-value to output, one-to-one correspondence to action in game.
+        """
+        super(FCN, self).__init__()
+        self.fc1 = nn.Linear(n_states, 128)
+        self.fc2 = nn.Linear(128, 128)
+        self.fc3 = nn.Linear(128, n_actions)
+        
+    def forward(self, x):
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        return self.fc3(x)

codes/dqn/plot.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 16:30:09
+@LastEditor: John
+@LastEditTime: 2020-06-14 11:38:42
+@Discription: 
+@Environment: python 3.7.7
+'''
+import matplotlib.pyplot as plt
+import numpy as np
+import os 
+
+def plot(item,ylabel='rewards'):
+    plt.figure()
+    plt.plot(np.arange(len(item)), item)
+    plt.title(ylabel+' of DQN') 
+    plt.ylabel('rewards')
+    plt.xlabel('episodes')
+    plt.savefig(os.path.dirname(__file__)+"/result/"+ylabel+".png")
+    plt.show()
+if __name__ == "__main__":
+
+    output_path = os.path.dirname(__file__)+"/result/"
+    rewards=np.load(output_path+"rewards.npy", )
+    moving_average_rewards=np.load(output_path+"moving_average_rewards.npy",)
+    plot(rewards)
+    plot(moving_average_rewards,ylabel='moving_average_rewards')