forked from andrew-j-levy/Hierarchical-Actor-Critc-HAC-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdesign_agent_and_env_reacher.py
159 lines (111 loc) · 8.32 KB
/
design_agent_and_env_reacher.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
"""
This file provides the template for designing the agent and environment. The below hyperparameters must be assigned to a value for the algorithm to work properly.
"""
import numpy as np
from environment import Environment
from utils import check_validity
from agent import Agent
def design_agent_and_env(FLAGS):
"""
1. DESIGN AGENT
The key hyperparameters for agent construction are
a. Number of levels in agent hierarchy
b. Max sequence length in which each policy will specialize
c. Max number of atomic actions allowed in an episode
d. Environment timesteps per atomic action
See Section 3 of this file for other agent hyperparameters that can be configured.
"""
FLAGS.layers = 2 # Enter number of levels in agent hierarchy
FLAGS.time_scale = 10 # Enter max sequence length in which each policy will specialize
# Enter max number of atomic actions. This will typically be FLAGS.time_scale**(FLAGS.layers). However,
# in the UR5 Reacher task, we use a shorter episode length.
max_actions = FLAGS.time_scale ** (FLAGS.layers - 1) * 6
timesteps_per_action = 15 # Provide the number of time steps per atomic action.
"""
2. DESIGN ENVIRONMENT
a. Designer must provide the original UMDP (S,A,T,G,R).
- The S,A,T components can be fulfilled by providing the Mujoco model.
- The user must separately specifiy the initial state space.
- G can be provided by specifying the end goal space.
- R, which by default uses a shortest path {-1,0} reward function, can be implemented by specifying two components: (i) a function that maps the state space to the end goal space and (ii) the end goal achievement thresholds for each dimensions of the end goal.
b. In order to convert the original UMDP into a hierarchy of k UMDPs, the designer must also provide
- The subgoal action space, A_i, for all higher-level UMDPs i > 0
- R_i for levels 0 <= i < k-1 (i.e., all levels that try to achieve goals in the subgoal space). As in the original UMDP, R_i can be implemented by providing two components:(i) a function that maps the state space to the subgoal space and (ii) the subgoal achievement thresholds.
c. Designer should also provide subgoal and end goal visualization functions in order to show video of training. These can be updated in "display_subgoal" and "display_end_goal" methods in the "environment.py" file.
"""
# Provide file name of Mujoco model(i.e., "pendulum.xml"). Make sure file is stored in "mujoco_files" folder
model_name = "reacher.xml"
# Provide initial state space consisting of the ranges for all joint angles and velocities. In the UR5 Reacher
# task, we use a random initial shoulder position and use fixed values for the remainder. Initial joint velocities
# are set to 0.
initial_joint_pos = np.array([0, 0]) # (2,)
initial_joint_pos = np.reshape(initial_joint_pos, (len(initial_joint_pos), 1)) # (2,1)
initial_joint_ranges = np.concatenate((initial_joint_pos, initial_joint_pos), 1) # (2,2)
initial_joint_ranges[0] = np.array([-np.pi, np.pi])
initial_joint_ranges[1] = np.array([-3, 3])
initial_state_space = np.concatenate((initial_joint_ranges, np.zeros((len(initial_joint_ranges), 2)), np.zeros((1, 2))), 0) # (5,2), joint ranges for joint0+joint1, vel range for joint0+1, and x,y range for fingertip
# Provide end goal space. The code supports two types of end goal spaces if user would like to train on a larger
# end goal space. If user needs to make additional customizations to the end goals, the "get_next_goal" method in
# "environment.py" can be updated.
# In the UR5 reacher environment, the end goal will be the desired joint positions for the 3 main joints.
goal_space_train = [[-.08, .08], [-.08, .08]]
goal_space_test = [[-.1, .1], [-.1, .1]]
# Provide a function that maps from the state space to the end goal space. This is used to determine whether the
# agent should be given the sparse reward. It is also used for Hindsight Experience Replay to determine which end
# goal was achieved after a sequence of actions.
# Supplementary function that will ensure all angles are between [-2*np.pi,2*np.pi]
def bound_angle(angle):
bounded_angle = np.absolute(angle) % (2 * np.pi)
if angle < 0:
bounded_angle = -bounded_angle
return bounded_angle
project_state_to_end_goal = lambda sim, state: np.array(sim.data.get_body_xpos("fingertip")[:2])
# Set end goal achievement thresholds. If the agent is within the threshold for each dimension, the end goal has
# been achieved and the reward of 0 is granted.
angle_threshold = np.deg2rad(10)
end_goal_thresholds = np.array([.03, .03])
# Provide range for each dimension of subgoal space in order to configure subgoal actor networks. Subgoal space can be the same as the state space or some other projection out of the state space. In our implementation of the UR5 reacher task, the subgoal space is the state space, which is the concatenation of all joint positions and joint velocities.
subgoal_bounds = np.array([[-2 * np.pi, 2 * np.pi], [-2 * np.pi, 2 * np.pi], [-4, 4], [-4, 4]])
# Provide state to subgoal projection function.
project_state_to_subgoal = lambda sim, state: np.concatenate((np.array(
[bound_angle(sim.data.qpos[i]) for i in range(len(sim.data.qpos))]), np.array(
[np.clip(sim.data.qvel[i], -4, 4) for i in range(len(sim.data.qvel))])))
# Set subgoal achievement thresholds
velo_threshold = 2
subgoal_thresholds = np.concatenate(
(np.array([angle_threshold for i in range(2)]), np.array([velo_threshold for i in range(2)])))
# To properly visualize goals, update "display_end_goal" and "display_subgoals" methods in "environment.py"
"""
3. SET MISCELLANEOUS HYPERPARAMETERS
Below are some other agent hyperparameters that can affect results, including
a. Subgoal testing percentage
b. Subgoal penalty
c. Exploration noise
d. Replay buffer size
"""
agent_params = {}
# Define percentage of actions that a subgoal level (i.e. level i > 0) will test subgoal actions
agent_params["subgoal_test_perc"] = 0.3
# Define subgoal penalty for missing subgoal. Please note that by default the Q value target for missed subgoals
# does not include Q-value of next state (i.e, discount rate = 0). As a result, the Q-value target for missed
# subgoal just equals penalty. For instance in this 3-level UR5 implementation, if a level proposes a subgoal
# and misses it, the Q target value for this action would be -10. To incorporate the next state in the penalty,
# go to the "penalize_subgoal" method in the "layer.py" file.
agent_params["subgoal_penalty"] = -FLAGS.time_scale
# Define exploration noise that is added to both subgoal actions and atomic actions. Noise added is Gaussian N(0, noise_percentage * action_dim_range)
agent_params["atomic_noise"] = [0.1 for i in range(2)]
agent_params["subgoal_noise"] = [0.01 for i in range(4)]
# Define number of episodes of transitions to be stored by each level of the hierarchy
agent_params["episodes_to_store"] = 500
# Provide training schedule for agent. Training by default will alternate between exploration and testing. Hyperparameter below indicates number of exploration episodes. Testing occurs for 100 episodes. To change number of testing episodes, go to "ran_HAC.py".
agent_params["num_exploration_episodes"] = 50
# For other relevant agent hyperparameters, please refer to the "agent.py" and "layer.py" files
# Ensure environment customization have been properly entered
check_validity(model_name, goal_space_train, goal_space_test, end_goal_thresholds, initial_state_space,
subgoal_bounds, subgoal_thresholds, max_actions, timesteps_per_action)
# Instantiate and return agent and environment
env = Environment(model_name, goal_space_train, goal_space_test, project_state_to_end_goal, end_goal_thresholds,
initial_state_space, subgoal_bounds, project_state_to_subgoal, subgoal_thresholds, max_actions,
timesteps_per_action, FLAGS.show)
agent = Agent(FLAGS, env, agent_params)
return agent, env