chore: 📝 Finished MADDPG: Add figures of the training results and modified hyperparams

Author: Phoenix-Shen

MultiAgentDeepDeterministicPolicyGradient/config.yaml

@@ -6,22 +6,22 @@ num_advs: 1
 
 # training hyperparameters
 lr_actor: 0.0001
-lr_critic: 0.001
+lr_critic: 0.003
 # exploration factor
 epsilon: 0.1
 noise_rate: 0.1
 # reward decay
 gamma: 0.95
 # soft update rate
-tau: 0.01
-cuda: False
+tau: 0.05
+cuda: True
 # experience
-batch_size: 256
+batch_size: 10000
 buffer_size: 500000
 
 # settings about saving models and logs
-log_dir: .\MultiAgentDeepDeterministicPolicyGradient\logs
-save_dir: .\MultiAgentDeepDeterministicPolicyGradient\models
+log_dir: ./MultiAgentDeepDeterministicPolicyGradient/logs
+save_dir: ./MultiAgentDeepDeterministicPolicyGradient/models
 
 # evaluation settings
 eval_interval: 1000

MultiAgentDeepDeterministicPolicyGradient/figs/al.jpg

@@ -24,13 +24,13 @@ def __init__(
         self.action_high = action_high
         self.action_dim = action_dim
         self.net = nn.Sequential(
-            nn.Linear(obs_dim, 256),
+            nn.Linear(obs_dim, 64),
             nn.ReLU(),
-            nn.Linear(256, 256),
+            nn.Linear(64, 64),
             nn.ReLU(),
-            nn.Linear(256, 256),
+            nn.Linear(64, 64),
             nn.ReLU(),
-            nn.Linear(256, action_dim),
+            nn.Linear(64, action_dim),
             nn.Tanh(),
         )
 
@@ -46,8 +46,7 @@ def select_action(
     ) -> np.ndarray:
         # random exploration
         if np.random.uniform() < epsilon:
-            mu = np.random.uniform(-self.action_high,
-                                   self.action_high, self.action_dim)
+            mu = np.random.uniform(-self.action_high, self.action_high, self.action_dim)
 
         else:
 
@@ -61,7 +60,7 @@ def select_action(
             noise = noise_rate * self.action_high * np.random.randn(*mu.shape)
             mu += noise
             mu = np.clip(mu, -self.action_high, self.action_high)
-        return mu
+        return mu.copy()
 
 
 class Critic(nn.Module):
@@ -75,13 +74,13 @@ def __init__(
         self.action_high = action_high
         # critic should give scores for all agents' actions
         self.net = nn.Sequential(
-            nn.Linear(sum(obs_dims) + sum(action_dims), 256),
+            nn.Linear(sum(obs_dims) + sum(action_dims), 64),
             nn.ReLU(),
-            nn.Linear(256, 256),
+            nn.Linear(64, 64),
             nn.ReLU(),
-            nn.Linear(256, 256),
+            nn.Linear(64, 64),
             nn.ReLU(),
-            nn.Linear(256, 1),
+            nn.Linear(64, 1),
         )
 
     def forward(self, obs: Tensor, actions: Tensor) -> Tensor:
@@ -153,27 +152,21 @@ def __init__(
         for i in range(n_agents):
             self.actors.append(Actor(action_high, obs_dims[i], action_dims[i]))
             self.critics.append(Critic(action_high, obs_dims, action_dims))
-            self.target_actors.append(
-                Actor(action_high, obs_dims[i], action_dims[i]))
-            self.target_critics.append(
-                Critic(action_high, obs_dims, action_dims))
+            self.target_actors.append(Actor(action_high, obs_dims[i], action_dims[i]))
+            self.target_critics.append(Critic(action_high, obs_dims, action_dims))
             # load_state_dict
             self.target_actors[i].load_state_dict(self.actors[i].state_dict())
-            self.target_critics[i].load_state_dict(
-                self.critics[i].state_dict())
+            self.target_critics[i].load_state_dict(self.critics[i].state_dict())
             # optimizers
-            self.optimizer_a.append(optim.Adam(
-                self.actors[i].parameters(), lr=lr_a))
-            self.optimizer_c.append(optim.Adam(
-                self.critics[i].parameters(), lr=lr_c))
+            self.optimizer_a.append(optim.Adam(self.actors[i].parameters(), lr=lr_a))
+            self.optimizer_c.append(optim.Adam(self.critics[i].parameters(), lr=lr_c))
 
             self.actors[i] = self.actors[i].to(self.device)
             self.critics[i] = self.critics[i].to(self.device)
             self.target_actors[i] = self.target_actors[i].to(self.device)
             self.target_critics[i] = self.target_critics[i].to(self.device)
 
-        self.buffer = MemoryBuffer(
-            mem_capacity, obs_dims, action_dims, self.n_agents)
+        self.buffer = MemoryBuffer(mem_capacity, obs_dims, action_dims, self.n_agents)
         self.writer = SummaryWriter(log_dir=log_dir)
 
     def learn(self):
@@ -260,40 +253,39 @@ def _update_policy(self, transitions: dict):
 
             # comput td target and use the square of td residual as the loss
             q_value = self.critics[i].forward(o, mu)
-            critic_loss = t.mean((q_target - q_value) *(q_target - q_value))
+            critic_loss = t.mean((q_target - q_value) * (q_target - q_value))
 
             # actor loss, Actor's goal is to make Critic's scoring higher
             mu[i] = self.actors[i].forward(o[i])
             actor_loss = -self.critics[i].forward(o, mu).mean()
 
             # then perform gradient descent
             self.optimizer_a[i].zero_grad()
-            self.optimizer_c[i].zero_grad()
-            critic_loss.backward()
             actor_loss.backward()
             self.optimizer_a[i].step()
+            self.optimizer_c[i].zero_grad()
+            critic_loss.backward()
             self.optimizer_c[i].step()
 
             actor_losses.append(actor_loss.item())
             critic_losses.append(critic_loss.item())
 
-        # then soft update the target network
-        self._soft_update_target()
+            # then soft update the target network
+            self._soft_update_target(i)
 
         return actor_losses, critic_losses
 
-    def _soft_update_target(self) -> None:
-        for i in range(self.n_agents):
-            for target_param, param in zip(
+    def _soft_update_target(self,i) -> None:
+        
+        for target_param, param in zip(
                 self.target_actors[i].parameters(), self.actors[i].parameters()
             ):
                 target_param.data.copy_(
                     (1 - self.tau) * target_param.data + self.tau * param.data
                 )
 
-            for target_param, param in zip(
-                self.target_critics[i].parameters(
-                ), self.critics[i].parameters()
+        for target_param, param in zip(
+                self.target_critics[i].parameters(), self.critics[i].parameters()
             ):
                 target_param.data.copy_(
                     (1 - self.tau) * target_param.data + self.tau * param.data

MultiAgentDeepDeterministicPolicyGradient/figs/cl.jpg

@@ -15,3 +15,11 @@
 ## 优化思路
 
 多Agent需要执行for循环，能否把它变成多线程呢，这样就可以大大加快速度了。
+
+## 结果
+
+![reward](./figs/reward.jpg)
+
+![aloss](./figs/al.jpg)
+
+![closs](./figs/cl.jpg)

MultiAgentDeepDeterministicPolicyGradient/figs/reward.jpg

@@ -3,7 +3,7 @@
 
 
 if __name__ == "__main__":
-    args = load_cfg(r"MultiAgentDeepDeterministicPolicyGradient\config.yaml")
+    args = load_cfg(r"MultiAgentDeepDeterministicPolicyGradient/config.yaml")
     env, args = make_env(args)
 
     agent = MADDPG(

MultiAgentDeepDeterministicPolicyGradient/models.py

@@ -532,7 +532,8 @@ def forward(self, x: t.Tensor) -> t.Tensor:
 
   Q_{\pi}(s_t,a_t) &= \mathbb{E}[U_t \vert S_t=s_t,A_t= a_t]\\
 
-  V_{\pi}(s_t)&=\mathbb{E}*A[Q*{\pi}(s_t,A)], A \sim \pi (\cdot \vert s_t)
+  V_{\pi}(s_t)&=\mathbb{E}_
+  A[Q_{\pi}(s_t,A)], A \sim \pi (\cdot \vert s_t)
 
   \end{aligned}
   $$