Add models.

Add some comments.

Author: peterparser

actor_network.h5

@@ -4,26 +4,26 @@
 
 class DQNAgent:
 
-    def __init__(self, hidden_layers, state_spec, action_spec, buffer, learning_rate, is_prioritized, is_double):
+    def __init__(self, hidden_layers, state_spec, action_spec, buffer, learning_rate, is_prioritized):
         self.buffer = buffer
-        self.training_network = tf.keras.models.Sequential()
-        self.training_network.add(tf.keras.layers.InputLayer(input_shape=(state_spec,)))
+        self.network = tf.keras.models.Sequential()
+        self.network.add(tf.keras.layers.InputLayer(input_shape=(state_spec,)))
 
         for hidden_layer in hidden_layers:
-            self.training_network.add(tf.keras.layers.Dense(hidden_layer, activation='relu'))
+            self.network.add(tf.keras.layers.Dense(hidden_layer, activation='relu'))
 
-        self.training_network.add(tf.keras.layers.Dense(action_spec, activation='linear'))
+        self.network.add(tf.keras.layers.Dense(action_spec, activation='linear'))
 
         self.optimizer = tf.keras.optimizers.Adam(learning_rate)
         self.is_prioritized = is_prioritized
-        self.is_double = is_double
         self.loss = tf.keras.losses.mean_squared_error
 
+    # Epsilon-greedy policy
     def play_action(self, state, epsilon):
         if np.random.random() < epsilon:
-            return np.random.choice(self.training_network.output_shape[1])
+            return np.random.choice(self.network.output_shape[1])
         else:
-            return np.argmax(self.training_network.predict(np.atleast_2d(state))[0])
+            return np.argmax(self.network.predict(np.atleast_2d(state))[0])
 
     def train(self, gamma, batch_size):
         if self.is_prioritized:
@@ -32,12 +32,12 @@ def train(self, gamma, batch_size):
         else:
             states, actions, rewards, states_next, dones = self.buffer.sample(batch_size)
 
-        target = rewards + (1 - dones) * gamma * np.max(self.training_network.predict(states_next), axis=1)
+        target = rewards + (1 - dones) * gamma * np.max(self.network.predict(states_next), axis=1)
 
         # Custom training loop taken by the teaching material on the course website
-        mask = tf.one_hot(actions, self.training_network.output_shape[1])
+        mask = tf.one_hot(actions, self.network.output_shape[1])
         with tf.GradientTape() as tape:
-            q_values = self.training_network(states)
+            q_values = self.network(states)
             predicted = tf.reduce_sum(q_values * mask, axis=1)
 
             if self.is_prioritized:
@@ -48,7 +48,7 @@ def train(self, gamma, batch_size):
         if self.is_prioritized:
             self.buffer.update_priority(ids, (target - predicted))
 
-        variables = self.training_network.trainable_variables
+        variables = self.network.trainable_variables
         gradients = tape.gradient(loss, variables)
         self.optimizer.apply_gradients(zip(gradients, variables))
 
@@ -78,13 +78,15 @@ def __init__(self, hidden_layers_actor, hidden_layers_critic, state_spec, action
         self.optimizer_critic = tf.keras.optimizers.Adam(learning_rate_critic)
         self.loss = tf.keras.losses.mean_squared_error
 
+    # Playing action by following the policy (output of the actor network)
     def play_action(self, state):
         probabilities = self.actor_network(np.atleast_2d(state))
         selection_probabilities = probabilities[0] / np.sum(probabilities[0])
         action = np.random.choice(self.actor_network.output_shape[1], p=selection_probabilities)
         return action
 
     def play_and_train(self, state, env, gamma):
+        # persistent needed because i will call tape.gradient 2 times one for the critic and one for the actor
         with tf.GradientTape(persistent=True) as tape:
             probabilities = self.actor_network(np.atleast_2d(state))
             # I need to normalize probability because numpy wants that the sum must be 1 and the softmax gives me

agents.py

@@ -24,11 +24,12 @@
 }
 
 
+# Training method for Actor critic
 def start_training_ac():
     env = gym.make(hyperparams['environment'])
     state_spec = len(env.observation_space.sample())
     action_spec = env.action_space.n
-    log_name = 'final'
+    log_name = 'final_build'
     log_dir = 'logs/acrobotAC/' + log_name
     log_writer = tf.summary.create_file_writer(log_dir)
 
@@ -49,21 +50,25 @@ def start_training_ac():
         total_rewards[episode] = episode_reward
         avg_rewards = total_rewards[max(0, episode - 20):(episode + 1)].mean()
         env.reset()
+
         with log_writer.as_default():
             tf.summary.scalar('episode reward', episode_reward, step=episode)
             tf.summary.scalar('avg for 20 episodes', avg_rewards, step=episode)
 
+    agent.actor_network.save_weights('actor_network.h5')
+
 
+# Training method for dqn
 def start_training_dqn(is_prioritized):
     if is_prioritized:
-        prio = "Prio"
+        prio = "with_priority"
     else:
-        prio = ""
+        prio = "no_priority"
 
     env = gym.make(hyperparams['environment'])
     state_spec = len(env.observation_space.sample())
     action_spec = env.action_space.n
-    log_name = 'final' + prio
+    log_name = 'final_build' + prio
     log_dir = 'logs/acrobot/' + log_name
 
     log_writer = tf.summary.create_file_writer(log_dir)
@@ -95,15 +100,12 @@ def start_training_dqn(is_prioritized):
 
         total_rewards[episode] = episode_reward
         avg_rewards = total_rewards[max(0, episode - 20):(episode + 1)].mean()
-
         env.reset()
-        if episode % 100 == 0:
-            print(avg_rewards)
 
         with log_writer.as_default():
             tf.summary.scalar('episode reward', episode_reward, step=episode)
             tf.summary.scalar('avg for 20 episodes', avg_rewards, step=episode)
-
+    agent.network.save_weights('dqn_{}_network.h5'.format(prio))
     env.close()
 
 
@@ -119,10 +121,10 @@ def test_model(model, is_ac):
         agent.actor_network.load_weights(model)
 
     else:
-        agent = DQNAgent(hyperparams['network'], state_spec, action_spec, buffer, hyperparams['learning_rate'],
+        agent = DQNAgent(hyperparams['hidden_layer_dqn'], state_spec, action_spec, buffer, hyperparams['learning_rate_dqn'],
                          is_prioritized)
 
-        agent.training_network.load_weights(model)
+        agent.network.load_weights(model)
     obs = env.reset()
     env.render()
     # Play 20 episodes
@@ -157,6 +159,7 @@ def test_model(model, is_ac):
     if args.mode == 'train':
         print('TRAIN')
         print("PER", args.per)
+        print("Actor critic", args.ac)
         if args.ac:
             start_training_ac()
         else: