feat: 🐛 fix some bugs in PPO and added some figures

Author: Phoenix-Shen

ProximalPolicyOptimization(PPO)/LunarLander.png

@@ -18,11 +18,11 @@ def __init__(self, n_features, n_actions) -> None:
         super().__init__()
 
         self.net = nn.Sequential(
-            nn.Linear(n_features, 128),
+            nn.Linear(n_features, 256),
             nn.ReLU(),
-            nn.Linear(128, 128),
+            nn.Linear(256, 256),
             nn.ReLU(),
-            nn.Linear(128, n_actions)
+            nn.Linear(256, n_actions)
         )
 
     def forward(self, observation: np.ndarray):
@@ -101,8 +101,9 @@ def learn(self, total_timesteps):
                     timesteps_so_far, i, actor_loss.item(), critic_loss.item()))
         # Step 8 Finally end for
             timesteps_so_far += np.sum(batch_lens)
+            ep_sum_rewards = [sum(rw) for rw in batch_r]
             self.sw.add_scalar("avg_reward", np.mean(
-                np.concatenate(batch_r)), timesteps_so_far)
+                ep_sum_rewards), timesteps_so_far)
     # the function to collect data
 
     def rollout(self):

ProximalPolicyOptimization(PPO)/lossBipedalWalker-v3.png

@@ -1,5 +1,13 @@
 # PPO onpolicy 算法 - OPENAI BASELINE
 
+BipedalWalker-v3
+
+![BipedalWalker-v3](./lossBipedalWalker-v3.png)
+
+LunarLanderContinuous-v2
+
+![lunarlander](./LunarLander.png)
+
 <https://medium.com/@eyyu/coding-ppo-from-scratch-with-pytorch-part-1-4-613dfc1b14c8>
 
 - 主要为了解决 actor critic 训练步长的问题，在这里实现了 ppo-clip
@@ -26,6 +34,8 @@ on：与环境交互的这个 agent 就是我们要学习的 agent，off：不
 
 效果很差，因为“偶尔的胜利”不足以使网络的参数完全修正，但是 offpolicy 的 dqn with per 能够多次学习成功的经验，所以对于这个 Pendulum-v1 来说，8 太行
 
+对于月球车来说也是不行的，他会走向local minimum（飞天上不下来）
+
 # TrustRegionPolicyOptimization
 
 - TRPO 算法 (Trust Region Policy Optimization)和 PPO 算法 (Proximal Policy Optimization)都属于 MM(Minorize-Maximizatio)算法

ProximalPolicyOptimization(PPO)/model.py

@@ -4,8 +4,8 @@
 import numpy as np
 from tensorboardX import SummaryWriter
 if __name__ == "__main__":
-    model_dir = "./ProximalPolicyOptimization(PPO)/saved_models/ACTOR 2021-12-14 16-45-2.pth"
-    writer = SummaryWriter("./ProximalPolicyOptimization/logs")
+    model_dir = "./ProximalPolicyOptimization(PPO)/saved_models/ACTOR 2022-5-20 15-54-15.pth"
+    writer = SummaryWriter("./ProximalPolicyOptimization(PPO)/logs")
     env = gym.make("Pendulum-v1")
     ppo = model.PPO(env, writer)
     ppo.actor.load_state_dict(t.load(model_dir))
@@ -18,6 +18,9 @@
         env.render()
         a = ppo.actor.forward(t.FloatTensor(s)).detach().numpy()
 
-        s_, r, _, _ = env.step(a)
+        s_, r, done, _ = env.step(a)
         s = s_
         print("action:{},reward:{}".format(a, r))
+
+        if done:
+            s = env.reset()

ProximalPolicyOptimization(PPO)/readme.md

@@ -2,7 +2,7 @@
 import model
 from tensorboardX import SummaryWriter
 writer = SummaryWriter("./ProximalPolicyOptimization(PPO)/logs")
-env = gym.make("LunarLanderContinuous-v2")
+env = gym.make("Pendulum-v1")
 ppo = model.PPO(env, writer)
-ppo.learn(1000000)
-ppo.save_model("./ProximalPolicyOptimization/saved_models")
+ppo.learn(1500000)
+ppo.save_model("./ProximalPolicyOptimization(PPO)/saved_models")