add DQN_cnn

codes/DQN_cnn/agent.py

@@ -0,0 +1,107 @@
+import random
+import math
+import torch
+import torch.optim as optim
+import torch.nn.functional as F
+from DQN_cnn.memory import ReplayBuffer
+from DQN_cnn.model import CNN
+
+
+class DQNcnn:
+    def __init__(self, screen_height,screen_width, action_dim, cfg):
+
+        self.device = cfg.device  
+        self.action_dim = action_dim
+        self.gamma = cfg.gamma
+        # e-greedy策略相关参数
+        self.actions_count = 0 
+        self.epsilon = 0
+        self.epsilon_start = cfg.epsilon_start
+        self.epsilon_end = cfg.epsilon_end
+        self.epsilon_decay = cfg.epsilon_decay
+        self.batch_size = cfg.batch_size
+        self.policy_net = CNN(screen_height, screen_width,
+                              action_dim).to(self.device)
+        self.target_net = CNN(screen_height, screen_width,
+                              action_dim).to(self.device)
+        self.target_net.load_state_dict(self.policy_net.state_dict()) # target_net的初始模型参数完全复制policy_net
+        self.target_net.eval()  # 不启用 BatchNormalization 和 Dropout
+        self.optimizer = optim.RMSprop(self.policy_net.parameters(),lr = cfg.lr) # 可查parameters()与state_dict()的区别，前者require_grad=True
+        self.loss = 0
+        self.memory = ReplayBuffer(cfg.memory_capacity)
+        
+
+    def choose_action(self, state):
+        '''选择动作
+        Args:
+            state [array]: [description]
+        Returns:
+            action [array]: [description]
+        '''
+        self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
+            math.exp(-1. * self.actions_count / self.epsilon_decay)
+        self.actions_count += 1
+        if random.random() > self.epsilon:   
+            with torch.no_grad():
+                q_value = self.policy_net(state) # q_value比如tensor([[-0.2522,  0.3887]])
+                # tensor.max(1)返回每行的最大值以及对应的下标，
+                # 如torch.return_types.max(values=tensor([10.3587]),indices=tensor([0]))
+                # 所以tensor.max(1)[1]返回最大值对应的下标，即action
+                action = q_value.max(1)[1].view(1, 1)  # 注意这里action是个张量，如tensor([1])
+                return action
+        else:
+            return torch.tensor([[random.randrange(self.action_dim)]], device=self.device, dtype=torch.long)
+
+    def update(self):
+        if len(self.memory) < self.batch_size:
+            return
+        transitions = self.memory.sample(self.batch_size)
+        # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
+        # detailed explanation). This converts batch-array of Transitions
+        # to Transition of batch-arrays.
+        batch = self.memory.Transition(*zip(*transitions))
+
+        # Compute a mask of non-final states and concatenate the batch elements
+        # (a final state would've been the one after which simulation ended)
+        non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,
+                                                batch.state_)), device=self.device, dtype=torch.bool)
+        
+        non_final_state_s = torch.cat([s for s in batch.state_
+                                           if s is not None])
+        state_batch = torch.cat(batch.state)   
+        action_batch = torch.cat(batch.action) 
+        reward_batch = torch.cat(batch.reward) # tensor([1., 1.,...,])
+        
+
+        # Compute Q(s_t, a) - the model computes Q(s_t), then we select the
+        # columns of actions taken. These are the actions which would've been taken
+        # for each batch state according to policy_net
+        state_action_values = self.policy_net(
+            state_batch).gather(1, action_batch) #tensor([[ 1.1217],...,[ 0.8314]])
+
+        # Compute V(s_{t+1}) for all next states.
+        # Expected values of actions for non_final_state_s are computed based
+        # on the "older" target_net; selecting their best reward with max(1)[0].
+        # This is merged based on the mask, such that we'll have either the expected
+        # state value or 0 in case the state was final.
+        state__values = torch.zeros(self.batch_size, device=self.device)
+
+        state__values[non_final_mask] = self.target_net(
+            non_final_state_s).max(1)[0].detach()
+
+        # Compute the expected Q values
+        expected_state_action_values = (state__values * self.gamma) + reward_batch # tensor([0.9685, 0.9683,...,])
+        
+        # Compute Huber loss
+        self.loss = F.smooth_l1_loss(
+            state_action_values, expected_state_action_values.unsqueeze(1))  # .unsqueeze增加一个维度
+        # Optimize the model
+        self.optimizer.zero_grad() # zero_grad clears old gradients from the last step (otherwise you’d just accumulate the gradients from all loss.backward() calls).
+        self.loss.backward() # loss.backward() computes the derivative of the loss w.r.t. the parameters (or anything requiring gradients) using backpropagation.
+        for param in self.policy_net.parameters(): # clip防止梯度爆炸
+            param.grad.data.clamp_(-1, 1)
+        self.optimizer.step() # causes the optimizer to take a step based on the gradients of the parameters.
+
+
+if __name__ == "__main__":
+    dqn = DQN()

codes/DQN_cnn/env.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 10:02:35
+@LastEditor: John
+@LastEditTime: 2020-06-11 16:57:34
+@Discription: 
+@Environment: python 3.7.7
+'''
+
+import numpy as np
+import torch
+import torchvision.transforms as T
+from PIL import Image
+
+resize = T.Compose([T.ToPILImage(),
+                    T.Resize(40, interpolation=Image.CUBIC),
+                    T.ToTensor()])
+
+
+def get_cart_location(env,screen_width):
+    world_width = env.x_threshold * 2
+    scale = screen_width / world_width
+    return int(env.state[0] * scale + screen_width / 2.0)  # MIDDLE OF CART
+
+def get_screen(env,device):
+    # Returned screen requested by gym is 400x600x3, but is sometimes larger
+    # such as 800x1200x3. Transpose it into torch order (CHW).
+    screen = env.render(mode='rgb_array').transpose((2, 0, 1))
+    # Cart is in the lower half, so strip off the top and bottom of the screen
+    _, screen_height, screen_width = screen.shape
+    screen = screen[:, int(screen_height*0.4):int(screen_height * 0.8)]
+    view_width = int(screen_width * 0.6)
+    cart_location = get_cart_location(env,screen_width)
+    if cart_location < view_width // 2:
+        slice_range = slice(view_width)
+    elif cart_location > (screen_width - view_width // 2):
+        slice_range = slice(-view_width, None)
+    else:
+        slice_range = slice(cart_location - view_width // 2,
+                            cart_location + view_width // 2)
+    # Strip off the edges, so that we have a square image centered on a cart
+    screen = screen[:, :, slice_range]
+    # Convert to float, rescale, convert to torch tensor
+    # (this doesn't require a copy)
+    screen = np.ascontiguousarray(screen, dtype=np.float32) / 255
+    screen = torch.from_numpy(screen)
+    # Resize, and add a batch dimension (BCHW)
+    return resize(screen).unsqueeze(0).to(device)
+
+if __name__ == "__main__":
+
+    import gym
+    env = gym.make('CartPole-v0').unwrapped
+    # if gpu is to be used
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    env.reset()
+    import matplotlib.pyplot as plt
+    
+    plt.figure()
+    plt.imshow(get_screen(env,device).cpu().squeeze(0).permute(1, 2, 0).numpy(),
+            interpolation='none')
+    plt.title('Example extracted screen')
+    plt.show()

codes/DQN_cnn/main.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 10:01:09
+@LastEditor: John
+LastEditTime: 2021-03-23 20:43:28
+@Discription: 
+@Environment: python 3.7.7
+'''
+import sys,os
+sys.path.append(os.getcwd()) # add current terminal path to sys.path
+import gym
+import torch
+import datetime
+from DQN_cnn.env import get_screen
+from DQN_cnn.agent import DQNcnn
+from common.plot import plot_rewards
+from common.utils import save_results
+
+sys.path.append(os.getcwd())  # add current terminal path to sys.path
+
+SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
+SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # path to save model
+if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): 
+    os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/")
+if not os.path.exists(SAVED_MODEL_PATH): 
+    os.mkdir(SAVED_MODEL_PATH)
+RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # path to save rewards
+if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): 
+    os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/")
+if not os.path.exists(RESULT_PATH): 
+    os.mkdir(RESULT_PATH)
+
+class DQNcnnConfig:
+    def __init__(self) -> None:
+        self.algo = "DQN_cnn"  # name of algo
+        self.gamma = 0.99
+        self.epsilon_start = 0.95  # e-greedy策略的初始epsilon
+        self.epsilon_end = 0.05
+        self.epsilon_decay = 200
+        self.lr = 0.01  # leanring rate
+        self.memory_capacity = 10000  # Replay Memory容量
+        self.batch_size = 64
+        self.train_eps = 250  # 训练的episode数目
+        self.train_steps = 200  # 训练每个episode的最大长度
+        self.target_update = 4  # target net的更新频率
+        self.eval_eps = 20  # 测试的episode数目
+        self.eval_steps = 200  # 测试每个episode的最大长度
+        self.hidden_dim = 128  # 神经网络隐藏层维度
+        self.device = torch.device(
+            "cuda" if torch.cuda.is_available() else "cpu")  # if gpu is to be used
+
+def train(cfg, env, agent):
+    rewards = []
+    ma_rewards = []
+    for i_episode in range(cfg.train_eps):
+        # Initialize the environment and state
+        env.reset()
+        last_screen = get_screen(env, cfg.device)
+        current_screen = get_screen(env, cfg.device)
+        state = current_screen - last_screen
+        ep_reward = 0
+        for i_step in range(cfg.train_steps+1):
+            # Select and perform an action
+            action = agent.choose_action(state)
+            _, reward, done, _ = env.step(action.item())
+            ep_reward += reward
+            reward = torch.tensor([reward], device=cfg.device)
+            # Observe new state
+            last_screen = current_screen
+            current_screen = get_screen(env, cfg.device)
+            if done:
+                break
+            state_ = current_screen - last_screen
+            # Store the transition in memory
+            agent.memory.push(state, action, state_, reward)
+            # Move to the next state
+            state = state_
+            # Perform one step of the optimization (on the target network)
+            agent.update()
+        # Update the target network, copying all weights and biases in DQN
+        if i_episode % cfg.target_update == 0:
+            agent.target_net.load_state_dict(agent.policy_net.state_dict())
+        print('Episode:{}/{}, Reward:{}, Steps:{}, Explore:{:.2f}, Done:{}'.format(i_episode+1,cfg.train_eps,ep_reward,i_step+1,agent.epsilon,done))
+        rewards.append(ep_reward)
+        if ma_rewards:
+            ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
+        else:
+            ma_rewards.append(ep_reward)
+    return rewards,ma_rewards
+
+
+if __name__ == "__main__":
+    cfg = DQNcnnConfig()
+    # Get screen size so that we can initialize layers correctly based on shape
+    # returned from AI gym. Typical dimensions at this point are close to 3x40x90
+    # which is the result of a clamped and down-scaled render buffer in get_screen(env,device)
+    # 因为这里环境的state需要从默认的向量改为图像，所以要unwrapped更改state
+    env = gym.make('CartPole-v0').unwrapped
+    env.reset()
+    init_screen = get_screen(env, cfg.device)
+    _, _, screen_height, screen_width = init_screen.shape
+    # Get number of actions from gym action space
+    action_dim = env.action_space.n
+    agent = DQNcnn(screen_height, screen_width,
+                   action_dim, cfg)
+    rewards,ma_rewards = train(cfg,env,agent)
+    save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH)
+    plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH)

codes/DQN_cnn/memory.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 09:42:44
+@LastEditor: John
+LastEditTime: 2021-03-23 20:38:41
+@Discription: 
+@Environment: python 3.7.7
+'''
+from collections import namedtuple
+import random
+            
+class ReplayBuffer(object):
+
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self.buffer = []
+        self.position = 0
+        self.Transition = namedtuple('Transition',
+                        ('state', 'action', 'state_', 'reward'))
+
+    def push(self, *args):
+        """Saves a transition."""
+        if len(self.buffer) < self.capacity:
+            self.buffer.append(None)
+        self.buffer[self.position] = self.Transition(*args)
+        self.position = (self.position + 1) % self.capacity
+
+    def sample(self, batch_size):
+        return random.sample(self.buffer, batch_size)
+
+    def __len__(self):
+        return len(self.buffer)

codes/DQN_cnn/model.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+@Author: John
+@Email: johnjim0816@gmail.com
+@Date: 2020-06-11 12:18:12
+@LastEditor: John
+@LastEditTime: 2020-06-11 17:23:45
+@Discription: 
+@Environment: python 3.7.7
+'''
+import torch.nn as nn
+import torch.nn.functional as F
+
+class CNN(nn.Module):
+
+    def __init__(self, h, w, n_outputs):
+        super(CNN, self).__init__()
+        self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2)
+        self.bn1 = nn.BatchNorm2d(16)
+        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2)
+        self.bn2 = nn.BatchNorm2d(32)
+        self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2)
+        self.bn3 = nn.BatchNorm2d(32)
+
+        # Number of Linear input connections depends on output of conv2d layers
+        # and therefore the input image size, so compute it.
+        def conv2d_size_out(size, kernel_size = 5, stride = 2):
+            return (size - (kernel_size - 1) - 1) // stride  + 1
+        convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(w)))
+        convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(h)))
+        linear_input_size = convw * convh * 32
+        self.head = nn.Linear(linear_input_size, n_outputs)
+
+    # Called with either one element to determine next action, or a batch
+    # during optimization. Returns tensor([[left0exp,right0exp]...]).
+    def forward(self, x):
+        x = F.relu(self.bn1(self.conv1(x)))
+        x = F.relu(self.bn2(self.conv2(x)))
+        x = F.relu(self.bn3(self.conv3(x)))
+        return self.head(x.view(x.size(0), -1))

codes/README.md

@@ -30,7 +30,7 @@ python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.18.0
 |        [Q-Learning](./QLearning)         |                                                             | [CliffWalking-v0](./envs/gym_info.md) |                                    |
 |             [Sarsa](./Sarsa)             |                                                             | [Racetrack](./envs/racetrack_env.md)  |                                    |
 |               [DQN](./DQN)               | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md)     |                                    |
-|                 DQN-cnn                  | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md)     | 与DQN相比使用了CNN而不是全链接网络 |
+|                 [DQN-cnn](./DQN_cnn)                  | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md)     | 与DQN相比使用了CNN而不是全链接网络 |
 |         [DoubleDQN](./DoubleDQN)         |                                                             | [CartPole-v0](./envs/gym_info.md)     |          效果不好，待改进          |
 |             Hierarchical DQN             |    [Hierarchical DQN](https://arxiv.org/abs/1604.06057)     |                                       |                                    |
 |    [PolicyGradient](./PolicyGradient)    |                                                             | [CartPole-v0](./envs/gym_info.md)     |                                    |