From 9cba78e3416d26db77afe9eee46dfef94e92e678 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Thu, 15 Jul 2021 11:36:23 -0700 Subject: [PATCH 01/19] Adding a prototype of multi-agents gym --- gym-unity/gym_unity/envs/ma_gym.py | 225 +++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 gym-unity/gym_unity/envs/ma_gym.py diff --git a/gym-unity/gym_unity/envs/ma_gym.py b/gym-unity/gym_unity/envs/ma_gym.py new file mode 100644 index 00000000000..6ee152ae705 --- /dev/null +++ b/gym-unity/gym_unity/envs/ma_gym.py @@ -0,0 +1,225 @@ +import numpy as np +from typing import Any, Dict, List, Optional, Tuple, Union + +import gym +from gym import error, spaces +from urllib.parse import urlparse, parse_qs + +from mlagents_envs.base_env import ActionTuple, BaseEnv +from mlagents_envs.base_env import DecisionSteps, TerminalSteps +from mlagents_envs.base_env import BaseEnv + + +def _parse_behavior(full_behavior): + parsed = urlparse(full_behavior) + name = parsed.path + ids = parse_qs(parsed.query) + team_id: int = 0 + if "team" in ids: + team_id = int(ids["team"][0]) + return name, team_id + + +class AgentStatus: + def __init__( + self, + behavior, + agent, + group, + group_reward, + action_mask, + interrupted, + current_obs, + reward, + done, + needs_action, + ): + self.behavior, self.team = _parse_behavior(behavior) + self.agent = agent + self.group = group + self.group_reward = group_reward + self.action_mask = action_mask + self.interrupted = interrupted + self.obs = current_obs + self.reward = reward + self.done = done + self.needs_action = needs_action + + +class MultiAgentGymWrapper(gym.Env): + def __init__(self, env: BaseEnv): + self._env = env + self._agent_index = -1 + self._current_batch: Optional[Tuple[DecisionSteps, TerminalSteps]] = None + self._behavior_index = 0 + self._current_action: Optional[ActionTuple] = None + self._last: Optional[AgentStatus] = None + + def _current_behavior_name(self) -> str: + return list(self._env.behavior_specs.keys())[self._behavior_index] + + @property + def last(self) -> Optional[AgentStatus]: + """ + This is not part of the original gym api. It is needed to know which behaviors + and agent ID the observation from a reset call belongs to. + """ + # TODO : info won't do. Info is supposed to be "extra" information, not needed + return self._last + + @property + def observation_space(self) -> spaces.Space: + current_behavior = self._current_behavior_name() + obs_spec = self._env.behavior_specs[current_behavior].observation_specs + obs_spaces = tuple( + [ + spaces.Box(-np.inf, np.inf, shape=spec.shape, dtype=np.float32) + for spec in obs_spec + ] + ) + return spaces.Tuple(obs_spaces) + + @property + def action_space(self) -> spaces.Space: + current_behavior = self._current_behavior_name() + act_spec = self._env.behavior_specs[current_behavior].action_spec + if act_spec.continuous_size == 0 and len(act_spec.discrete_branches) == 0: + raise Exception("No actions found") + if act_spec.discrete_size == 1: + d_space = spaces.Discrete(act_spec.discrete_branches[0]) + if act_spec.continuous_size == 0: + return d_space + if act_spec.discrete_size > 0: + d_space = spaces.MultiDiscrete(act_spec.discrete_branches) + if act_spec.continuous_size == 0: + return d_space + if act_spec.continuous_size > 0: + c_space = spaces.Box(-1, 1, (act_spec.continuous_size,), dtype=np.int32) + if len(act_spec.discrete_branches) == 0: + return c_space + return spaces.Tuple(tuple(c_space, d_space)) + + def step( + self, action: Any = None + ) -> Tuple[List[np.array], float, bool, Dict[str, Any]]: + # get one agent at a time. In single agent envs, behaves just like Gym. No need for gym wrapper + # return List[np.array] (observation), float (reward), bool (done), info (dict with group reward, group_id, behavior_name, agent_id, etc...) + + # Convert Actions + if action is not None: + if not self.action_space.contains(action): + raise Exception( + f"Invalid action, got {action} but was expecting action from {self.action_space}" + ) + if isinstance(self.action_space, spaces.Tuple): + action = ActionTuple(action[0], action[1]) + elif isinstance(self.action_space, spaces.MultiDiscrete): + action = ActionTuple(None, action) + elif isinstance(self.action_space, spaces.Discrete): + action = ActionTuple(None, np.array(action).reshape(1, 1)) + else: + action = ActionTuple(action, None) + + if self._current_batch is None: + raise Exception( + "You must reset the environment before you can perform a step" + ) + + decision_batch, termination_batch = self._current_batch + + # Build action + if self._current_action is None: + a_spec = self._env.behavior_specs[self._current_behavior_name()].action_spec + self._current_action = ActionTuple( + np.zeros( + (len(decision_batch), a_spec.continuous_size), dtype=np.float32 + ), + np.zeros( + (len(decision_batch), len(a_spec.discrete_branches)), dtype=np.int32 + ), + ) + if action is not None: + if self._agent_index >= 0 and self._agent_index < len(decision_batch): + + if action.continuous is not None: + self._current_action.continuous[ + self._agent_index + ] = action.continuous[0] + if action.discrete is not None: + self._current_action.discrete[self._agent_index] = action.discrete[ + 0 + ] + else: + # A Useless action was passed (the last.agent was done) + pass + + self._agent_index += 1 + + if self._agent_index < len(decision_batch): + + # the index is within the decsion steps + obs = [batch_obs[self._agent_index] for batch_obs in decision_batch.obs] + reward = decision_batch.reward[self._agent_index] + done = False + + self._last = AgentStatus( + list(self._env.behavior_specs.keys())[self._behavior_index], + decision_batch.agent_id[self._agent_index], + decision_batch.group_id[self._agent_index], + decision_batch.group_reward[self._agent_index], + [mask[self._agent_index] for mask in decision_batch.action_mask] + if decision_batch.action_mask is not None + else None, + False, + obs, + reward, + done, + True, + ) + + return obs, reward, done, None + + if self._agent_index < len(decision_batch) + len(termination_batch): + # The index is within the terminal steps + index = self._agent_index - len(decision_batch) + obs = [batch_obs[index] for batch_obs in termination_batch.obs] + reward = termination_batch.reward[index] + done = True + self._last = AgentStatus( + list(self._env.behavior_specs.keys())[self._behavior_index], + termination_batch.agent_id[index], + termination_batch.group_id[index], + termination_batch.group_reward[index], + None, + termination_batch.interrupted[index], + obs, + reward, + done, + False, + ) + + return obs, reward, done, None + # The index is too high, time to set the action for the agents we have + if self._current_action is not None: + self._env.set_actions(self._current_behavior_name(), self._current_action) + self._current_action = None + self._behavior_index += 1 + if self._behavior_index >= len(self._env.behavior_specs): + self._env.step() + self._behavior_index = 0 + self._current_batch = self._env.get_steps(self._current_behavior_name()) + self._agent_index = -1 + + return self.step() + + def reset(self) -> List[np.array]: + self._env.reset() + self._agent_index = -1 + self._behavior_index = 0 + self._current_batch = self._env.get_steps(self._current_behavior_name()) + self._current_action = None + obs, _, _, _ = self.step() + return obs + + def close(self) -> None: + self._env.close() From 2d0d7fb6ec34642ab29268fb2edef1b01b485161 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Thu, 15 Jul 2021 16:09:28 -0700 Subject: [PATCH 02/19] - --- gym-unity/gym_unity/envs/ma_gym.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gym-unity/gym_unity/envs/ma_gym.py b/gym-unity/gym_unity/envs/ma_gym.py index 6ee152ae705..91fdd917e80 100644 --- a/gym-unity/gym_unity/envs/ma_gym.py +++ b/gym-unity/gym_unity/envs/ma_gym.py @@ -158,7 +158,7 @@ def step( if self._agent_index < len(decision_batch): # the index is within the decsion steps - obs = [batch_obs[self._agent_index] for batch_obs in decision_batch.obs] + obs = tuple([batch_obs[self._agent_index] for batch_obs in decision_batch.obs]) reward = decision_batch.reward[self._agent_index] done = False @@ -182,7 +182,7 @@ def step( if self._agent_index < len(decision_batch) + len(termination_batch): # The index is within the terminal steps index = self._agent_index - len(decision_batch) - obs = [batch_obs[index] for batch_obs in termination_batch.obs] + obs = tuple([batch_obs[index] for batch_obs in termination_batch.obs]) reward = termination_batch.reward[index] done = True self._last = AgentStatus( From add8829c9b41108333a09e057ac2ea4ae1e67f8f Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Sat, 17 Jul 2021 09:52:16 -0700 Subject: [PATCH 03/19] adding 3DBall to the gym registry --- gym-unity/gym_unity/__init__.py | 2 + gym-unity/gym_unity/envs/__init__.py | 365 +-------------------------- gym-unity/setup.py | 2 +- 3 files changed, 12 insertions(+), 357 deletions(-) diff --git a/gym-unity/gym_unity/__init__.py b/gym-unity/gym_unity/__init__.py index 7902085efe7..6240e4b221f 100644 --- a/gym-unity/gym_unity/__init__.py +++ b/gym-unity/gym_unity/__init__.py @@ -3,3 +3,5 @@ # Git tag that will be checked to determine whether to trigger upload to pypi __release_tag__ = None + +import gym_unity.envs diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index 322d83659bb..b18be395b8a 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -1,361 +1,14 @@ -import itertools -import numpy as np -from typing import Any, Dict, List, Optional, Tuple, Union +from gym.envs.registration import register +from mlagents_envs.registry import default_registry +from gym_unity.envs.ma_gym import MultiAgentGymWrapper -import gym -from gym import error, spaces -from mlagents_envs.base_env import ActionTuple, BaseEnv -from mlagents_envs.base_env import DecisionSteps, TerminalSteps -from mlagents_envs import logging_util +def gym_entry_point(env_name, **kwargs): + def create(**kwagrs): + _e = default_registry[env_name].make(**kwargs) + return MultiAgentGymWrapper(_e) + return create -class UnityGymException(error.Error): - """ - Any error related to the gym wrapper of ml-agents. - """ - pass - - -logger = logging_util.get_logger(__name__) -logging_util.set_log_level(logging_util.INFO) - -GymStepResult = Tuple[np.ndarray, float, bool, Dict] - - -class UnityToGymWrapper(gym.Env): - """ - Provides Gym wrapper for Unity Learning Environments. - """ - - def __init__( - self, - unity_env: BaseEnv, - uint8_visual: bool = False, - flatten_branched: bool = False, - allow_multiple_obs: bool = False, - action_space_seed: Optional[int] = None, - ): - """ - Environment initialization - :param unity_env: The Unity BaseEnv to be wrapped in the gym. Will be closed when the UnityToGymWrapper closes. - :param uint8_visual: Return visual observations as uint8 (0-255) matrices instead of float (0.0-1.0). - :param flatten_branched: If True, turn branched discrete action spaces into a Discrete space rather than - MultiDiscrete. - :param allow_multiple_obs: If True, return a list of np.ndarrays as observations with the first elements - containing the visual observations and the last element containing the array of vector observations. - If False, returns a single np.ndarray containing either only a single visual observation or the array of - vector observations. - :param action_space_seed: If non-None, will be used to set the random seed on created gym.Space instances. - """ - self._env = unity_env - - # Take a single step so that the brain information will be sent over - if not self._env.behavior_specs: - self._env.step() - - self.visual_obs = None - - # Save the step result from the last time all Agents requested decisions. - self._previous_decision_step: DecisionSteps = None - self._flattener = None - # Hidden flag used by Atari environments to determine if the game is over - self.game_over = False - self._allow_multiple_obs = allow_multiple_obs - - # Check brain configuration - if len(self._env.behavior_specs) != 1: - raise UnityGymException( - "There can only be one behavior in a UnityEnvironment " - "if it is wrapped in a gym." - ) - - self.name = list(self._env.behavior_specs.keys())[0] - self.group_spec = self._env.behavior_specs[self.name] - - if self._get_n_vis_obs() == 0 and self._get_vec_obs_size() == 0: - raise UnityGymException( - "There are no observations provided by the environment." - ) - - if not self._get_n_vis_obs() >= 1 and uint8_visual: - logger.warning( - "uint8_visual was set to true, but visual observations are not in use. " - "This setting will not have any effect." - ) - else: - self.uint8_visual = uint8_visual - if ( - self._get_n_vis_obs() + self._get_vec_obs_size() >= 2 - and not self._allow_multiple_obs - ): - logger.warning( - "The environment contains multiple observations. " - "You must define allow_multiple_obs=True to receive them all. " - "Otherwise, only the first visual observation (or vector observation if" - "there are no visual observations) will be provided in the observation." - ) - - # Check for number of agents in scene. - self._env.reset() - decision_steps, _ = self._env.get_steps(self.name) - self._check_agents(len(decision_steps)) - self._previous_decision_step = decision_steps - - # Set action spaces - if self.group_spec.action_spec.is_discrete(): - self.action_size = self.group_spec.action_spec.discrete_size - branches = self.group_spec.action_spec.discrete_branches - if self.group_spec.action_spec.discrete_size == 1: - self._action_space = spaces.Discrete(branches[0]) - else: - if flatten_branched: - self._flattener = ActionFlattener(branches) - self._action_space = self._flattener.action_space - else: - self._action_space = spaces.MultiDiscrete(branches) - - elif self.group_spec.action_spec.is_continuous(): - if flatten_branched: - logger.warning( - "The environment has a non-discrete action space. It will " - "not be flattened." - ) - - self.action_size = self.group_spec.action_spec.continuous_size - high = np.array([1] * self.group_spec.action_spec.continuous_size) - self._action_space = spaces.Box(-high, high, dtype=np.float32) - else: - raise UnityGymException( - "The gym wrapper does not provide explicit support for both discrete " - "and continuous actions." - ) - - if action_space_seed is not None: - self._action_space.seed(action_space_seed) - - # Set observations space - list_spaces: List[gym.Space] = [] - shapes = self._get_vis_obs_shape() - for shape in shapes: - if uint8_visual: - list_spaces.append(spaces.Box(0, 255, dtype=np.uint8, shape=shape)) - else: - list_spaces.append(spaces.Box(0, 1, dtype=np.float32, shape=shape)) - if self._get_vec_obs_size() > 0: - # vector observation is last - high = np.array([np.inf] * self._get_vec_obs_size()) - list_spaces.append(spaces.Box(-high, high, dtype=np.float32)) - if self._allow_multiple_obs: - self._observation_space = spaces.Tuple(list_spaces) - else: - self._observation_space = list_spaces[0] # only return the first one - - def reset(self) -> Union[List[np.ndarray], np.ndarray]: - """Resets the state of the environment and returns an initial observation. - Returns: observation (object/list): the initial observation of the - space. - """ - self._env.reset() - decision_step, _ = self._env.get_steps(self.name) - n_agents = len(decision_step) - self._check_agents(n_agents) - self.game_over = False - - res: GymStepResult = self._single_step(decision_step) - return res[0] - - def step(self, action: List[Any]) -> GymStepResult: - """Run one timestep of the environment's dynamics. When end of - episode is reached, you are responsible for calling `reset()` - to reset this environment's state. - Accepts an action and returns a tuple (observation, reward, done, info). - Args: - action (object/list): an action provided by the environment - Returns: - observation (object/list): agent's observation of the current environment - reward (float/list) : amount of reward returned after previous action - done (boolean/list): whether the episode has ended. - info (dict): contains auxiliary diagnostic information. - """ - if self.game_over: - raise UnityGymException( - "You are calling 'step()' even though this environment has already " - "returned done = True. You must always call 'reset()' once you " - "receive 'done = True'." - ) - if self._flattener is not None: - # Translate action into list - action = self._flattener.lookup_action(action) - - action = np.array(action).reshape((1, self.action_size)) - - action_tuple = ActionTuple() - if self.group_spec.action_spec.is_continuous(): - action_tuple.add_continuous(action) - else: - action_tuple.add_discrete(action) - self._env.set_actions(self.name, action_tuple) - - self._env.step() - decision_step, terminal_step = self._env.get_steps(self.name) - self._check_agents(max(len(decision_step), len(terminal_step))) - if len(terminal_step) != 0: - # The agent is done - self.game_over = True - return self._single_step(terminal_step) - else: - return self._single_step(decision_step) - - def _single_step(self, info: Union[DecisionSteps, TerminalSteps]) -> GymStepResult: - if self._allow_multiple_obs: - visual_obs = self._get_vis_obs_list(info) - visual_obs_list = [] - for obs in visual_obs: - visual_obs_list.append(self._preprocess_single(obs[0])) - default_observation = visual_obs_list - if self._get_vec_obs_size() >= 1: - default_observation.append(self._get_vector_obs(info)[0, :]) - else: - if self._get_n_vis_obs() >= 1: - visual_obs = self._get_vis_obs_list(info) - default_observation = self._preprocess_single(visual_obs[0][0]) - else: - default_observation = self._get_vector_obs(info)[0, :] - - if self._get_n_vis_obs() >= 1: - visual_obs = self._get_vis_obs_list(info) - self.visual_obs = self._preprocess_single(visual_obs[0][0]) - - done = isinstance(info, TerminalSteps) - - return (default_observation, info.reward[0], done, {"step": info}) - - def _preprocess_single(self, single_visual_obs: np.ndarray) -> np.ndarray: - if self.uint8_visual: - return (255.0 * single_visual_obs).astype(np.uint8) - else: - return single_visual_obs - - def _get_n_vis_obs(self) -> int: - result = 0 - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 3: - result += 1 - return result - - def _get_vis_obs_shape(self) -> List[Tuple]: - result: List[Tuple] = [] - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 3: - result.append(obs_spec.shape) - return result - - def _get_vis_obs_list( - self, step_result: Union[DecisionSteps, TerminalSteps] - ) -> List[np.ndarray]: - result: List[np.ndarray] = [] - for obs in step_result.obs: - if len(obs.shape) == 4: - result.append(obs) - return result - - def _get_vector_obs( - self, step_result: Union[DecisionSteps, TerminalSteps] - ) -> np.ndarray: - result: List[np.ndarray] = [] - for obs in step_result.obs: - if len(obs.shape) == 2: - result.append(obs) - return np.concatenate(result, axis=1) - - def _get_vec_obs_size(self) -> int: - result = 0 - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 1: - result += obs_spec.shape[0] - return result - - def render(self, mode="rgb_array"): - """ - Return the latest visual observations. - Note that it will not render a new frame of the environment. - """ - return self.visual_obs - - def close(self) -> None: - """Override _close in your subclass to perform any necessary cleanup. - Environments will automatically close() themselves when - garbage collected or when the program exits. - """ - self._env.close() - - def seed(self, seed: Any = None) -> None: - """Sets the seed for this env's random number generator(s). - Currently not implemented. - """ - logger.warning("Could not seed environment %s", self.name) - return - - @staticmethod - def _check_agents(n_agents: int) -> None: - if n_agents > 1: - raise UnityGymException( - f"There can only be one Agent in the environment but {n_agents} were detected." - ) - - @property - def metadata(self): - return {"render.modes": ["rgb_array"]} - - @property - def reward_range(self) -> Tuple[float, float]: - return -float("inf"), float("inf") - - @property - def action_space(self) -> gym.Space: - return self._action_space - - @property - def observation_space(self): - return self._observation_space - - -class ActionFlattener: - """ - Flattens branched discrete action spaces into single-branch discrete action spaces. - """ - - def __init__(self, branched_action_space): - """ - Initialize the flattener. - :param branched_action_space: A List containing the sizes of each branch of the action - space, e.g. [2,3,3] for three branches with size 2, 3, and 3 respectively. - """ - self._action_shape = branched_action_space - self.action_lookup = self._create_lookup(self._action_shape) - self.action_space = spaces.Discrete(len(self.action_lookup)) - - @classmethod - def _create_lookup(self, branched_action_space): - """ - Creates a Dict that maps discrete actions (scalars) to branched actions (lists). - Each key in the Dict maps to one unique set of branched actions, and each value - contains the List of branched actions. - """ - possible_vals = [range(_num) for _num in branched_action_space] - all_actions = [list(_action) for _action in itertools.product(*possible_vals)] - # Dict should be faster than List for large action spaces - action_lookup = { - _scalar: _action for (_scalar, _action) in enumerate(all_actions) - } - return action_lookup - - def lookup_action(self, action): - """ - Convert a scalar discrete action into a unique set of branched actions. - :param: action: A scalar value representing one of the discrete actions. - :return: The List containing the branched actions. - """ - return self.action_lookup[action] +register(id="3DBall-v0", entry_point=gym_entry_point("3DBall")) diff --git a/gym-unity/setup.py b/gym-unity/setup.py index 354b5e8b26f..4bcb975dcb5 100755 --- a/gym-unity/setup.py +++ b/gym-unity/setup.py @@ -38,6 +38,6 @@ def run(self): author_email="ML-Agents@unity3d.com", url="https://github.com/Unity-Technologies/ml-agents", packages=find_packages(), - install_requires=["gym", f"mlagents_envs=={VERSION}"], + install_requires=["gym", f"mlagents_envs=={VERSION}", "certifi"], cmdclass={"verify": VerifyVersionCommand}, ) From ef7d8182d7b5ff813ad1c872d56af51a6f0020f9 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Sat, 17 Jul 2021 21:49:51 -0700 Subject: [PATCH 04/19] Adding all registry environments --- gym-unity/gym_unity/envs/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index b18be395b8a..94fc3277eae 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -10,5 +10,5 @@ def create(**kwagrs): return create - -register(id="3DBall-v0", entry_point=gym_entry_point("3DBall")) +for key in default_registry: + register(id=key + "-v0", entry_point=gym_entry_point(key)) From ff061639bb04915d5b69cb9e8f1ef659bf3552c8 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Mon, 19 Jul 2021 09:26:38 -0700 Subject: [PATCH 05/19] add a try-catch --- gym-unity/gym_unity/envs/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index 94fc3277eae..32e0fe7160c 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -11,4 +11,8 @@ def create(**kwagrs): return create for key in default_registry: - register(id=key + "-v0", entry_point=gym_entry_point(key)) + registry_key = key + "-v0" + try: + register(id=registry_key, entry_point=gym_entry_point(key)) + except: + pass From 3f87cc67285d7a0114f83f22a51de7623e62b8f5 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Mon, 19 Jul 2021 16:20:45 -0700 Subject: [PATCH 06/19] Bug fix. Made sure Q-LEarning worked --- gym-unity/gym_unity/envs/ma_gym.py | 51 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/gym-unity/gym_unity/envs/ma_gym.py b/gym-unity/gym_unity/envs/ma_gym.py index 91fdd917e80..71b6ee587dc 100644 --- a/gym-unity/gym_unity/envs/ma_gym.py +++ b/gym-unity/gym_unity/envs/ma_gym.py @@ -155,33 +155,9 @@ def step( self._agent_index += 1 - if self._agent_index < len(decision_batch): - - # the index is within the decsion steps - obs = tuple([batch_obs[self._agent_index] for batch_obs in decision_batch.obs]) - reward = decision_batch.reward[self._agent_index] - done = False - - self._last = AgentStatus( - list(self._env.behavior_specs.keys())[self._behavior_index], - decision_batch.agent_id[self._agent_index], - decision_batch.group_id[self._agent_index], - decision_batch.group_reward[self._agent_index], - [mask[self._agent_index] for mask in decision_batch.action_mask] - if decision_batch.action_mask is not None - else None, - False, - obs, - reward, - done, - True, - ) - - return obs, reward, done, None - - if self._agent_index < len(decision_batch) + len(termination_batch): + if self._agent_index < len(termination_batch): # The index is within the terminal steps - index = self._agent_index - len(decision_batch) + index = self._agent_index obs = tuple([batch_obs[index] for batch_obs in termination_batch.obs]) reward = termination_batch.reward[index] done = True @@ -197,8 +173,31 @@ def step( done, False, ) + return obs, reward, done, None + if self._agent_index < len(decision_batch) + len(termination_batch): + index = self._agent_index - len(termination_batch) + # the index is within the decsion steps + obs = tuple([batch_obs[index] for batch_obs in decision_batch.obs]) + reward = decision_batch.reward[index] + done = False + + self._last = AgentStatus( + list(self._env.behavior_specs.keys())[self._behavior_index], + decision_batch.agent_id[index], + decision_batch.group_id[index], + decision_batch.group_reward[index], + [mask[index] for mask in decision_batch.action_mask] + if decision_batch.action_mask is not None + else None, + False, + obs, + reward, + done, + True, + ) return obs, reward, done, None + # The index is too high, time to set the action for the agents we have if self._current_action is not None: self._env.set_actions(self._current_behavior_name(), self._current_action) From d65438375aecfcc95da14ca830318f993e826043 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Fri, 23 Jul 2021 13:39:13 -0700 Subject: [PATCH 07/19] improvements --- gym-unity/gym_unity/envs/__init__.py | 4 +- .../gym_unity/envs/{ma_gym.py => gym_env.py} | 59 +++++++++++++++---- 2 files changed, 48 insertions(+), 15 deletions(-) rename gym-unity/gym_unity/envs/{ma_gym.py => gym_env.py} (77%) diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index 32e0fe7160c..0b2661a611f 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -1,12 +1,12 @@ from gym.envs.registration import register from mlagents_envs.registry import default_registry -from gym_unity.envs.ma_gym import MultiAgentGymWrapper +from gym_unity.envs.gym_env import UnityToGymWrapper def gym_entry_point(env_name, **kwargs): def create(**kwagrs): _e = default_registry[env_name].make(**kwargs) - return MultiAgentGymWrapper(_e) + return UnityToGymWrapper(_e) return create diff --git a/gym-unity/gym_unity/envs/ma_gym.py b/gym-unity/gym_unity/envs/gym_env.py similarity index 77% rename from gym-unity/gym_unity/envs/ma_gym.py rename to gym-unity/gym_unity/envs/gym_env.py index 71b6ee587dc..3aed88507a0 100644 --- a/gym-unity/gym_unity/envs/ma_gym.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -1,6 +1,7 @@ import numpy as np from typing import Any, Dict, List, Optional, Tuple, Union +import itertools import gym from gym import error, spaces from urllib.parse import urlparse, parse_qs @@ -46,20 +47,27 @@ def __init__( self.needs_action = needs_action -class MultiAgentGymWrapper(gym.Env): - def __init__(self, env: BaseEnv): +class UnityToGymWrapper(gym.Env): + def __init__(self, env: BaseEnv, action_space_seed: Optional[int] = None): self._env = env self._agent_index = -1 self._current_batch: Optional[Tuple[DecisionSteps, TerminalSteps]] = None self._behavior_index = 0 self._current_action: Optional[ActionTuple] = None self._last: Optional[AgentStatus] = None + self._action_spaces: Dict[str, spaces.Space] = {} + self._obs_spaces: Dict[str, spaces.Space] = {} + self._action_sampling_seed = action_space_seed def _current_behavior_name(self) -> str: - return list(self._env.behavior_specs.keys())[self._behavior_index] + names = list(self._env.behavior_specs.keys()) + if len(names) > self._behavior_index: + return names[self._behavior_index] + return None + @property - def last(self) -> Optional[AgentStatus]: + def active(self) -> Optional[AgentStatus]: """ This is not part of the original gym api. It is needed to know which behaviors and agent ID the observation from a reset call belongs to. @@ -68,8 +76,12 @@ def last(self) -> Optional[AgentStatus]: return self._last @property - def observation_space(self) -> spaces.Space: + def observation_space(self) -> Optional[spaces.Space]: current_behavior = self._current_behavior_name() + if current_behavior is None: + return None + if current_behavior in self._obs_spaces: + return self._obs_spaces[current_behavior] obs_spec = self._env.behavior_specs[current_behavior].observation_specs obs_spaces = tuple( [ @@ -77,27 +89,46 @@ def observation_space(self) -> spaces.Space: for spec in obs_spec ] ) - return spaces.Tuple(obs_spaces) + if len(obs_spaces) == 1: + self._obs_spaces[current_behavior] = obs_spaces[0] + else: + self._obs_spaces[current_behavior] = spaces.Tuple(obs_spaces) + return self._obs_spaces[current_behavior] @property - def action_space(self) -> spaces.Space: + def action_space(self) -> Optional[spaces.Space]: current_behavior = self._current_behavior_name() + if current_behavior is None: + return None + if current_behavior in self._action_spaces: + return self._action_spaces[current_behavior] act_spec = self._env.behavior_specs[current_behavior].action_spec if act_spec.continuous_size == 0 and len(act_spec.discrete_branches) == 0: raise Exception("No actions found") if act_spec.discrete_size == 1: d_space = spaces.Discrete(act_spec.discrete_branches[0]) + if self._action_sampling_seed is not None: + d_space.seed(self._action_sampling_seed) if act_spec.continuous_size == 0: - return d_space + self._action_spaces[current_behavior] = d_space + return self._action_spaces[current_behavior] if act_spec.discrete_size > 0: d_space = spaces.MultiDiscrete(act_spec.discrete_branches) + if self._action_sampling_seed is not None: + d_space.seed(self._action_sampling_seed) if act_spec.continuous_size == 0: - return d_space + self._action_spaces[current_behavior] = d_space + return self._action_spaces[current_behavior] if act_spec.continuous_size > 0: c_space = spaces.Box(-1, 1, (act_spec.continuous_size,), dtype=np.int32) + if self._action_sampling_seed is not None: + c_space.seed(self._action_sampling_seed) if len(act_spec.discrete_branches) == 0: - return c_space - return spaces.Tuple(tuple(c_space, d_space)) + self._action_spaces[current_behavior] = c_space + return self._action_spaces[current_behavior] + self._action_spaces[current_behavior] = spaces.Tuple(tuple(c_space, d_space)) + return self._action_spaces[current_behavior] + def step( self, action: Any = None @@ -158,7 +189,8 @@ def step( if self._agent_index < len(termination_batch): # The index is within the terminal steps index = self._agent_index - obs = tuple([batch_obs[index] for batch_obs in termination_batch.obs]) + obs = [batch_obs[index] for batch_obs in termination_batch.obs] + obs = obs if len(obs) > 1 else obs[0] reward = termination_batch.reward[index] done = True self._last = AgentStatus( @@ -178,7 +210,8 @@ def step( if self._agent_index < len(decision_batch) + len(termination_batch): index = self._agent_index - len(termination_batch) # the index is within the decsion steps - obs = tuple([batch_obs[index] for batch_obs in decision_batch.obs]) + obs = [batch_obs[index] for batch_obs in decision_batch.obs] + obs = obs if len(obs) > 1 else obs[0] reward = decision_batch.reward[index] done = False From 5e44c037c3a5f498d154683874eca160b7a814ac Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Fri, 23 Jul 2021 15:43:01 -0700 Subject: [PATCH 08/19] Writing brand new tests. Needs more --- gym-unity/gym_unity/envs/__init__.py | 7 +- gym-unity/gym_unity/envs/gym_env.py | 8 +- gym-unity/gym_unity/tests/test_gym.py | 292 +++----------------------- 3 files changed, 37 insertions(+), 270 deletions(-) diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index 0b2661a611f..c0e7f97f865 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -1,12 +1,13 @@ from gym.envs.registration import register from mlagents_envs.registry import default_registry from gym_unity.envs.gym_env import UnityToGymWrapper +from typing import Any, Dict, List, Optional -def gym_entry_point(env_name, **kwargs): - def create(**kwagrs): +def gym_entry_point(env_name): + def create(action_space_seed: Optional[int] = None, **kwargs): _e = default_registry[env_name].make(**kwargs) - return UnityToGymWrapper(_e) + return UnityToGymWrapper(_e, action_space_seed) return create diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py index 3aed88507a0..dc664c559db 100644 --- a/gym-unity/gym_unity/envs/gym_env.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -1,6 +1,7 @@ import numpy as np from typing import Any, Dict, List, Optional, Tuple, Union +import atexit import itertools import gym from gym import error, spaces @@ -49,6 +50,7 @@ def __init__( class UnityToGymWrapper(gym.Env): def __init__(self, env: BaseEnv, action_space_seed: Optional[int] = None): + # atexit.register(self.close) self._env = env self._agent_index = -1 self._current_batch: Optional[Tuple[DecisionSteps, TerminalSteps]] = None @@ -254,4 +256,8 @@ def reset(self) -> List[np.array]: return obs def close(self) -> None: - self._env.close() + if self._env is not None: + self._env.close() + + def __del__(self): + self.close() diff --git a/gym-unity/gym_unity/tests/test_gym.py b/gym-unity/gym_unity/tests/test_gym.py index 6928faa0fd2..805ec56a975 100644 --- a/gym-unity/gym_unity/tests/test_gym.py +++ b/gym-unity/gym_unity/tests/test_gym.py @@ -1,277 +1,37 @@ -from unittest import mock import pytest -import numpy as np -from gym import spaces -from gym_unity.envs import UnityToGymWrapper -from mlagents_envs.base_env import ( - BehaviorSpec, - ActionSpec, - DecisionSteps, - TerminalSteps, - BehaviorMapping, -) -from mlagents.trainers.tests.dummy_config import create_observation_specs_with_shapes +import gym +import gym_unity -def test_gym_wrapper(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec() - mock_decision_step, mock_terminal_step = create_mock_vector_steps(mock_spec) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - env = UnityToGymWrapper(mock_env) - assert isinstance(env.reset(), np.ndarray) - actions = env.action_space.sample() - assert actions.shape[0] == 2 - obs, rew, done, info = env.step(actions) +@pytest.mark.parametrize("env_name", ["3DBall-v0", "WallJump-v0", "GridWorld-v0"]) +def test_env(env_name): + env = gym.make(env_name) + obs = env.reset() assert env.observation_space.contains(obs) - assert isinstance(obs, np.ndarray) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - -def test_branched_flatten(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - vector_action_space_type="discrete", vector_action_space_size=[2, 2, 3] - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, num_agents=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, flatten_branched=True) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 12 - assert env._flattener.lookup_action(0) == [0, 0, 0] - assert env._flattener.lookup_action(11) == [1, 1, 2] - - # Check that False produces a MultiDiscrete - env = UnityToGymWrapper(mock_env, flatten_branched=False) - assert isinstance(env.action_space, spaces.MultiDiscrete) - - -def test_action_space(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - vector_action_space_type="discrete", vector_action_space_size=[5] - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, num_agents=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, flatten_branched=True) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 5 - - env = UnityToGymWrapper(mock_env, flatten_branched=False) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 5 - - -def test_action_space_seed(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec() - mock_decision_step, mock_terminal_step = create_mock_vector_steps(mock_spec) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - actions = [] - for _ in range(0, 2): - env = UnityToGymWrapper(mock_env, action_space_seed=1337) - env.reset() - actions.append(env.action_space.sample()) - assert (actions[0] == actions[1]).all() - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_visual(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=1, vector_observation_space_size=0 - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8) - assert isinstance(env.observation_space, spaces.Box) - assert isinstance(env.reset(), np.ndarray) - actions = env.action_space.sample() - assert actions.shape[0] == 2 - obs, rew, done, info = env.step(actions) + for _ in range(10): + obs, _, _, _ = env.step(env.action_space.sample()) + assert env.observation_space.contains(obs) + obs = env.reset() assert env.observation_space.contains(obs) - assert isinstance(obs, np.ndarray) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_single_visual_and_vector(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=1, - vector_observation_space_size=3, - vector_action_space_size=[2], - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=True) - assert isinstance(env.observation_space, spaces.Tuple) - assert len(env.observation_space) == 2 - reset_obs = env.reset() - assert isinstance(reset_obs, list) - assert len(reset_obs) == 2 - assert all(isinstance(ob, np.ndarray) for ob in reset_obs) - assert reset_obs[-1].shape == (3,) - assert len(reset_obs[0].shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, list) - assert len(obs) == 2 - assert all(isinstance(ob, np.ndarray) for ob in obs) - assert reset_obs[-1].shape == (3,) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - # check behavior for allow_multiple_obs = False - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=False) - assert isinstance(env.observation_space, spaces.Box) - reset_obs = env.reset() - assert isinstance(reset_obs, np.ndarray) - assert len(reset_obs.shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, np.ndarray) - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_multi_visual_and_vector(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=2, - vector_observation_space_size=3, - vector_action_space_size=[2], - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=2 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=True) - assert isinstance(env.observation_space, spaces.Tuple) - assert len(env.observation_space) == 3 - reset_obs = env.reset() - assert isinstance(reset_obs, list) - assert len(reset_obs) == 3 - assert all(isinstance(ob, np.ndarray) for ob in reset_obs) - assert reset_obs[-1].shape == (3,) - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert all(isinstance(ob, np.ndarray) for ob in obs) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - # check behavior for allow_multiple_obs = False - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=False) - assert isinstance(env.observation_space, spaces.Box) - reset_obs = env.reset() - assert isinstance(reset_obs, np.ndarray) - assert len(reset_obs.shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, np.ndarray) - - -# Helper methods - - -def create_mock_group_spec( - number_visual_observations=0, - vector_action_space_type="continuous", - vector_observation_space_size=3, - vector_action_space_size=None, -): - """ - Creates a mock BrainParameters object with parameters. - """ - # Avoid using mutable object as default param - if vector_action_space_type == "continuous": - if vector_action_space_size is None: - vector_action_space_size = 2 - else: - vector_action_space_size = vector_action_space_size[0] - action_spec = ActionSpec.create_continuous(vector_action_space_size) - else: - if vector_action_space_size is None: - vector_action_space_size = (2,) - else: - vector_action_space_size = tuple(vector_action_space_size) - action_spec = ActionSpec.create_discrete(vector_action_space_size) - obs_shapes = [(vector_observation_space_size,)] - for _ in range(number_visual_observations): - obs_shapes += [(8, 8, 3)] - obs_spec = create_observation_specs_with_shapes(obs_shapes) - return BehaviorSpec(obs_spec, action_spec) - + for _ in range(10): + obs, _, _, _ = env.step(env.action_space.sample()) + assert env.observation_space.contains(obs) + assert isinstance(env.action_space, gym.spaces.Space) + assert isinstance(env.observation_space, gym.spaces.Space) + env.close() -def create_mock_vector_steps(specs, num_agents=1, number_visual_observations=0): - """ - Creates a mock BatchedStepResult with vector observations. Imitates constant - vector observations, rewards, dones, and agents. + env = gym.make(env_name, action_space_seed=42) + env.reset() + rng_1 = env.action_space._np_random + env.close() + env.action_space - :BehaviorSpecs specs: The BehaviorSpecs for this mock - :int num_agents: Number of "agents" to imitate in your BatchedStepResult values. - """ - obs = [np.array([num_agents * [1, 2, 3]]).reshape(num_agents, 3)] - if number_visual_observations: - obs += [ - np.zeros(shape=(num_agents, 8, 8, 3), dtype=np.float32) - ] * number_visual_observations - rewards = np.array(num_agents * [1.0]) - agents = np.array(range(0, num_agents)) - group_id = np.array(num_agents * [0]) - group_rewards = np.array(num_agents * [0.0]) - return ( - DecisionSteps(obs, rewards, agents, None, group_id, group_rewards), - TerminalSteps.empty(specs), - ) + env = gym.make(env_name, action_space_seed=42) + env.reset() + rng_2 = env.action_space._np_random + env.close() + assert rng_1.randint(10000000000000) == rng_2.randint(10000000000000) -def setup_mock_unityenvironment(mock_env, mock_spec, mock_decision, mock_termination): - """ - Takes a mock UnityEnvironment and adds the appropriate properties, defined by the mock - GroupSpec and BatchedStepResult. - :Mock mock_env: A mock UnityEnvironment, usually empty. - :Mock mock_spec: An AgentGroupSpec object that specifies the params of this environment. - :Mock mock_decision: A DecisionSteps object that will be returned at each step and reset. - :Mock mock_termination: A TerminationSteps object that will be returned at each step and reset. - """ - mock_env.behavior_specs = BehaviorMapping({"MockBrain": mock_spec}) - mock_env.get_steps.return_value = (mock_decision, mock_termination) From 253510ba3e33472e1763b6572884ea337fdad4a5 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Mon, 26 Jul 2021 10:50:59 -0700 Subject: [PATCH 09/19] removing __del__ method --- gym-unity/gym_unity/envs/gym_env.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py index dc664c559db..487a8bf4f71 100644 --- a/gym-unity/gym_unity/envs/gym_env.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -259,5 +259,3 @@ def close(self) -> None: if self._env is not None: self._env.close() - def __del__(self): - self.close() From 7fa9e47cbcf664ad261f2783472fba33b221cc13 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Mon, 26 Jul 2021 14:48:58 -0700 Subject: [PATCH 10/19] Hacking the side channel to the gym interface, probably not a great idea since I need to access private fields --- gym-unity/gym_unity/envs/__init__.py | 12 ++++++++++++ gym-unity/gym_unity/envs/gym_env.py | 21 ++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index c0e7f97f865..2bfdea21352 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -4,8 +4,20 @@ from typing import Any, Dict, List, Optional +from mlagents_envs.side_channel.engine_configuration_channel import ( + EngineConfigurationChannel, +) +from mlagents_envs.side_channel.stats_side_channel import ( + StatsSideChannel, +) +from mlagents_envs.side_channel.environment_parameters_channel import ( + EnvironmentParametersChannel, +) + def gym_entry_point(env_name): def create(action_space_seed: Optional[int] = None, **kwargs): + if "side_channels" not in kwargs: + kwargs["side_channels"] = [EngineConfigurationChannel(), EnvironmentParametersChannel(), StatsSideChannel()] _e = default_registry[env_name].make(**kwargs) return UnityToGymWrapper(_e, action_space_seed) diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py index 487a8bf4f71..0e87845adfe 100644 --- a/gym-unity/gym_unity/envs/gym_env.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -60,6 +60,7 @@ def __init__(self, env: BaseEnv, action_space_seed: Optional[int] = None): self._action_spaces: Dict[str, spaces.Space] = {} self._obs_spaces: Dict[str, spaces.Space] = {} self._action_sampling_seed = action_space_seed + self._side_channel_dict = {type(v).__name__: v for v in self._env._side_channel_manager._side_channels_dict.values()} def _current_behavior_name(self) -> str: names = list(self._env.behavior_specs.keys()) @@ -67,6 +68,9 @@ def _current_behavior_name(self) -> str: return names[self._behavior_index] return None + def _assert_loaded(self) -> None: + if self._env is None: + raise Exception("No environment loaded") @property def active(self) -> Optional[AgentStatus]: @@ -75,10 +79,12 @@ def active(self) -> Optional[AgentStatus]: and agent ID the observation from a reset call belongs to. """ # TODO : info won't do. Info is supposed to be "extra" information, not needed + self._assert_loaded() return self._last @property def observation_space(self) -> Optional[spaces.Space]: + self._assert_loaded() current_behavior = self._current_behavior_name() if current_behavior is None: return None @@ -99,6 +105,7 @@ def observation_space(self) -> Optional[spaces.Space]: @property def action_space(self) -> Optional[spaces.Space]: + self._assert_loaded() current_behavior = self._current_behavior_name() if current_behavior is None: return None @@ -131,13 +138,22 @@ def action_space(self) -> Optional[spaces.Space]: self._action_spaces[current_behavior] = spaces.Tuple(tuple(c_space, d_space)) return self._action_spaces[current_behavior] + @property + def reward_range(self) -> Tuple[float, float]: + self._assert_loaded() + return - float("inf"), float("inf") + + @property + def side_channel(self) -> Dict[str, Any]: + self._assert_loaded() + return self._side_channel_dict def step( self, action: Any = None ) -> Tuple[List[np.array], float, bool, Dict[str, Any]]: # get one agent at a time. In single agent envs, behaves just like Gym. No need for gym wrapper # return List[np.array] (observation), float (reward), bool (done), info (dict with group reward, group_id, behavior_name, agent_id, etc...) - + self._assert_loaded() # Convert Actions if action is not None: if not self.action_space.contains(action): @@ -247,6 +263,7 @@ def step( return self.step() def reset(self) -> List[np.array]: + self._assert_loaded() self._env.reset() self._agent_index = -1 self._behavior_index = 0 @@ -256,6 +273,8 @@ def reset(self) -> List[np.array]: return obs def close(self) -> None: + self._assert_loaded() if self._env is not None: self._env.close() + self._env = None From c1e7f48f7ebc3c5cb629c8552f6b84d3172e2cbc Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 09:56:00 -0700 Subject: [PATCH 11/19] Some edits --- gym-unity/gym_unity/envs/gym_env.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py index 0e87845adfe..28b53f96c44 100644 --- a/gym-unity/gym_unity/envs/gym_env.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -50,7 +50,7 @@ def __init__( class UnityToGymWrapper(gym.Env): def __init__(self, env: BaseEnv, action_space_seed: Optional[int] = None): - # atexit.register(self.close) + atexit.register(self.close) self._env = env self._agent_index = -1 self._current_batch: Optional[Tuple[DecisionSteps, TerminalSteps]] = None @@ -273,8 +273,10 @@ def reset(self) -> List[np.array]: return obs def close(self) -> None: - self._assert_loaded() if self._env is not None: self._env.close() self._env = None + def __del__(self) -> None: + self.close() + From 34fb2e56015a18b413baa107cb74dbf27ab957ff Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 10:32:45 -0700 Subject: [PATCH 12/19] formatting --- gym-unity/gym_unity/__init__.py | 2 +- gym-unity/gym_unity/envs/__init__.py | 18 ++++++++----- gym-unity/gym_unity/envs/gym_env.py | 38 ++++++++++++--------------- gym-unity/gym_unity/tests/test_gym.py | 5 +--- gym-unity/setup.py | 6 ++--- 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/gym-unity/gym_unity/__init__.py b/gym-unity/gym_unity/__init__.py index 6240e4b221f..1ca31fe0ca5 100644 --- a/gym-unity/gym_unity/__init__.py +++ b/gym-unity/gym_unity/__init__.py @@ -4,4 +4,4 @@ # Git tag that will be checked to determine whether to trigger upload to pypi __release_tag__ = None -import gym_unity.envs +import gym_unity.envs # noqa diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index 2bfdea21352..df645bb498d 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -1,31 +1,35 @@ from gym.envs.registration import register +from gym.error import Error from mlagents_envs.registry import default_registry from gym_unity.envs.gym_env import UnityToGymWrapper -from typing import Any, Dict, List, Optional from mlagents_envs.side_channel.engine_configuration_channel import ( EngineConfigurationChannel, ) -from mlagents_envs.side_channel.stats_side_channel import ( - StatsSideChannel, -) +from mlagents_envs.side_channel.stats_side_channel import StatsSideChannel from mlagents_envs.side_channel.environment_parameters_channel import ( EnvironmentParametersChannel, ) + def gym_entry_point(env_name): - def create(action_space_seed: Optional[int] = None, **kwargs): + def create(action_space_seed=None, **kwargs): if "side_channels" not in kwargs: - kwargs["side_channels"] = [EngineConfigurationChannel(), EnvironmentParametersChannel(), StatsSideChannel()] + kwargs["side_channels"] = [ + EngineConfigurationChannel(), + EnvironmentParametersChannel(), + StatsSideChannel(), + ] _e = default_registry[env_name].make(**kwargs) return UnityToGymWrapper(_e, action_space_seed) return create + for key in default_registry: registry_key = key + "-v0" try: register(id=registry_key, entry_point=gym_entry_point(key)) - except: + except Error: pass diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py index 28b53f96c44..68318ebcc6b 100644 --- a/gym-unity/gym_unity/envs/gym_env.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -1,15 +1,13 @@ import numpy as np -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple import atexit -import itertools import gym from gym import error, spaces from urllib.parse import urlparse, parse_qs from mlagents_envs.base_env import ActionTuple, BaseEnv from mlagents_envs.base_env import DecisionSteps, TerminalSteps -from mlagents_envs.base_env import BaseEnv def _parse_behavior(full_behavior): @@ -60,17 +58,20 @@ def __init__(self, env: BaseEnv, action_space_seed: Optional[int] = None): self._action_spaces: Dict[str, spaces.Space] = {} self._obs_spaces: Dict[str, spaces.Space] = {} self._action_sampling_seed = action_space_seed - self._side_channel_dict = {type(v).__name__: v for v in self._env._side_channel_manager._side_channels_dict.values()} + self._side_channel_dict = { + type(v).__name__: v + for v in self._env._side_channel_manager._side_channels_dict.values() + } - def _current_behavior_name(self) -> str: + def _current_behavior_name(self) -> Optional[str]: names = list(self._env.behavior_specs.keys()) if len(names) > self._behavior_index: return names[self._behavior_index] return None def _assert_loaded(self) -> None: - if self._env is None: - raise Exception("No environment loaded") + if self._env is None or self._current_behavior_name() is None: + raise error.Error("No environment loaded") @property def active(self) -> Optional[AgentStatus]: @@ -92,10 +93,8 @@ def observation_space(self) -> Optional[spaces.Space]: return self._obs_spaces[current_behavior] obs_spec = self._env.behavior_specs[current_behavior].observation_specs obs_spaces = tuple( - [ - spaces.Box(-np.inf, np.inf, shape=spec.shape, dtype=np.float32) - for spec in obs_spec - ] + spaces.Box(-np.inf, np.inf, shape=spec.shape, dtype=np.float32) + for spec in obs_spec ) if len(obs_spaces) == 1: self._obs_spaces[current_behavior] = obs_spaces[0] @@ -113,7 +112,7 @@ def action_space(self) -> Optional[spaces.Space]: return self._action_spaces[current_behavior] act_spec = self._env.behavior_specs[current_behavior].action_spec if act_spec.continuous_size == 0 and len(act_spec.discrete_branches) == 0: - raise Exception("No actions found") + raise error.Error("No actions found") if act_spec.discrete_size == 1: d_space = spaces.Discrete(act_spec.discrete_branches[0]) if self._action_sampling_seed is not None: @@ -135,13 +134,13 @@ def action_space(self) -> Optional[spaces.Space]: if len(act_spec.discrete_branches) == 0: self._action_spaces[current_behavior] = c_space return self._action_spaces[current_behavior] - self._action_spaces[current_behavior] = spaces.Tuple(tuple(c_space, d_space)) + self._action_spaces[current_behavior] = spaces.Tuple((c_space, d_space)) return self._action_spaces[current_behavior] @property def reward_range(self) -> Tuple[float, float]: self._assert_loaded() - return - float("inf"), float("inf") + return -float("inf"), float("inf") @property def side_channel(self) -> Dict[str, Any]: @@ -150,14 +149,12 @@ def side_channel(self) -> Dict[str, Any]: def step( self, action: Any = None - ) -> Tuple[List[np.array], float, bool, Dict[str, Any]]: - # get one agent at a time. In single agent envs, behaves just like Gym. No need for gym wrapper - # return List[np.array] (observation), float (reward), bool (done), info (dict with group reward, group_id, behavior_name, agent_id, etc...) + ) -> Tuple[List[np.array], float, bool, Optional[Dict[str, Any]]]: self._assert_loaded() # Convert Actions if action is not None: - if not self.action_space.contains(action): - raise Exception( + if not self.action_space.contains(action): # type: ignore + raise error.Error( f"Invalid action, got {action} but was expecting action from {self.action_space}" ) if isinstance(self.action_space, spaces.Tuple): @@ -170,7 +167,7 @@ def step( action = ActionTuple(action, None) if self._current_batch is None: - raise Exception( + raise error.Error( "You must reset the environment before you can perform a step" ) @@ -279,4 +276,3 @@ def close(self) -> None: def __del__(self) -> None: self.close() - diff --git a/gym-unity/gym_unity/tests/test_gym.py b/gym-unity/gym_unity/tests/test_gym.py index 805ec56a975..2b1008c60e7 100644 --- a/gym-unity/gym_unity/tests/test_gym.py +++ b/gym-unity/gym_unity/tests/test_gym.py @@ -1,7 +1,7 @@ import pytest import gym -import gym_unity +import gym_unity # noqa @pytest.mark.parametrize("env_name", ["3DBall-v0", "WallJump-v0", "GridWorld-v0"]) @@ -32,6 +32,3 @@ def test_env(env_name): rng_2 = env.action_space._np_random env.close() assert rng_1.randint(10000000000000) == rng_2.randint(10000000000000) - - - diff --git a/gym-unity/setup.py b/gym-unity/setup.py index 4bcb975dcb5..b234ed3bc59 100755 --- a/gym-unity/setup.py +++ b/gym-unity/setup.py @@ -4,10 +4,10 @@ import sys from setuptools import setup, find_packages from setuptools.command.install import install -import gym_unity +from gym_unity import __version__, __release_tag__ -VERSION = gym_unity.__version__ -EXPECTED_TAG = gym_unity.__release_tag__ +VERSION = __version__ +EXPECTED_TAG = __release_tag__ class VerifyVersionCommand(install): From 2b5b23547ac73147932f2823c8f0570089930f57 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 10:39:22 -0700 Subject: [PATCH 13/19] hacking around the setup --- gym-unity/gym_unity/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gym-unity/gym_unity/__init__.py b/gym-unity/gym_unity/__init__.py index 1ca31fe0ca5..fb995500a8d 100644 --- a/gym-unity/gym_unity/__init__.py +++ b/gym-unity/gym_unity/__init__.py @@ -4,4 +4,7 @@ # Git tag that will be checked to determine whether to trigger upload to pypi __release_tag__ = None -import gym_unity.envs # noqa +try: + import gym_unity.envs # noqa +except ImportError: + pass From 2b8d38a4d20b79d7cc3b521b6cb4cff6fe45a0d7 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 11:55:36 -0700 Subject: [PATCH 14/19] fix test : nographics in tests --- gym-unity/gym_unity/envs/gym_env.py | 2 +- gym-unity/gym_unity/tests/test_gym.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py index 68318ebcc6b..91b0c41946e 100644 --- a/gym-unity/gym_unity/envs/gym_env.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -70,7 +70,7 @@ def _current_behavior_name(self) -> Optional[str]: return None def _assert_loaded(self) -> None: - if self._env is None or self._current_behavior_name() is None: + if self._env is None: raise error.Error("No environment loaded") @property diff --git a/gym-unity/gym_unity/tests/test_gym.py b/gym-unity/gym_unity/tests/test_gym.py index 2b1008c60e7..df1f3c6588e 100644 --- a/gym-unity/gym_unity/tests/test_gym.py +++ b/gym-unity/gym_unity/tests/test_gym.py @@ -4,9 +4,9 @@ import gym_unity # noqa -@pytest.mark.parametrize("env_name", ["3DBall-v0", "WallJump-v0", "GridWorld-v0"]) +@pytest.mark.parametrize("env_name", ["3DBall-v0", "WallJump-v0", "PushBlock-v0"]) def test_env(env_name): - env = gym.make(env_name) + env = gym.make(env_name, no_graphics=True, base_port=6000) obs = env.reset() assert env.observation_space.contains(obs) for _ in range(10): @@ -21,13 +21,12 @@ def test_env(env_name): assert isinstance(env.observation_space, gym.spaces.Space) env.close() - env = gym.make(env_name, action_space_seed=42) + env = gym.make(env_name, no_graphics=True, action_space_seed=42) env.reset() rng_1 = env.action_space._np_random env.close() - env.action_space - env = gym.make(env_name, action_space_seed=42) + env = gym.make(env_name, no_graphics=True, action_space_seed=42) env.reset() rng_2 = env.action_space._np_random env.close() From a7f4917a42d1f0c45a03f2d78c0b1fd2843d20d4 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 18:14:18 -0700 Subject: [PATCH 15/19] Modifying the make method so we can search for available ports when creating the environment --- gym-unity/gym_unity/envs/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index df645bb498d..2ba5b99a6f0 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -11,6 +11,7 @@ from mlagents_envs.side_channel.environment_parameters_channel import ( EnvironmentParametersChannel, ) +from mlagents_envs.exception import UnityWorkerInUseException def gym_entry_point(env_name): @@ -21,7 +22,18 @@ def create(action_space_seed=None, **kwargs): EnvironmentParametersChannel(), StatsSideChannel(), ] - _e = default_registry[env_name].make(**kwargs) + _e = None + if "base_port" not in kwargs: + port = 6000 + while _e is None: + try: + kwargs["base_port"] = port + _e = default_registry[env_name].make(**kwargs) + except UnityWorkerInUseException: + port += 1 + pass + else: + _e = default_registry[env_name].make(**kwargs) return UnityToGymWrapper(_e, action_space_seed) return create From 18905f1160a8dacc434474dcccc98f270686fc9d Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 18:47:15 -0700 Subject: [PATCH 16/19] edits to the markdown --- gym-unity/README.md | 46 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/gym-unity/README.md b/gym-unity/README.md index f664c16e467..21ef03e39f4 100755 --- a/gym-unity/README.md +++ b/gym-unity/README.md @@ -31,42 +31,34 @@ from the root of the project repository use: ```python from gym_unity.envs import UnityToGymWrapper -env = UnityToGymWrapper(unity_env, uint8_visual, flatten_branched, allow_multiple_obs) +env = UnityToGymWrapper(unity_env) ``` - `unity_env` refers to the Unity environment to be wrapped. -- `uint8_visual` refers to whether to output visual observations as `uint8` - values (0-255). Many common Gym environments (e.g. Atari) do this. By default - they will be floats (0.0-1.0). Defaults to `False`. +- `action_space_seed` is the optional seed for action sampling. If non-None, will + be used to set the random seed on created gym.Space instances. -- `flatten_branched` will flatten a branched discrete action space into a Gym - Discrete. Otherwise, it will be converted into a MultiDiscrete. Defaults to - `False`. +The returned environment `env` will function as a multi-agent gym. -- `allow_multiple_obs` will return a list of observations. The first elements - contain the visual observations and the last element contains the array of - vector observations. If False the environment returns a single array (containing - a single visual observations, if present, otherwise the vector observation). - Defaults to `False`. +Alternatively, you can get one of the environments from the Unity registry with the following code : -- `action_space_seed` is the optional seed for action sampling. If non-None, will - be used to set the random seed on created gym.Space instances. +``` +import gym +import gym_unity -The returned environment `env` will function as a gym. +env = gym.make("GridWorld-v0") +... +env.close() + +``` + +> **:warning: IMPORTANT** +> Note that a Unity environment can have **ANY** number of different agents all requesting decisions at different and irregular times. When calling `env.step()` the next observation returned can be for a completely different agent than the one the action was for. +To know which agent is currently in need of a decision, you can access the current agent identifier by calling `env.active.agent`. Calling `env.step()` will not necessarily move the simulation forward: `env.step()` will set the action for the `env.active.agent`. When all agents that needed an action received one, the simulation will move forward. ## Limitations -- It is only possible to use an environment with a **single** Agent. -- By default, the first visual observation is provided as the `observation`, if - present. Otherwise, vector observations are provided. You can receive all - visual and vector observations by using the `allow_multiple_obs=True` option in - the gym parameters. If set to `True`, you will receive a list of `observation` - instead of only one. -- The `TerminalSteps` or `DecisionSteps` output from the environment can still - be accessed from the `info` provided by `env.step(action)`. -- Stacked vector observations are not supported. -- Environment registration for use with `gym.make()` is currently not supported. - Calling env.render() will not render a new frame of the environment. It will return the latest visual observation if using visual observations. @@ -80,6 +72,8 @@ using these algorithms. This requires the creation of custom training scripts to launch each algorithm. In most cases these scripts can be created by making slight modifications to the ones provided for Atari and Mujoco environments. +Note that Baselines assumes that there is a single agent in the environment. Baselines will not work on environments where the number of agents is not strictly one. + These examples were tested with baselines version 0.1.6. ### Example - DQN Baseline @@ -274,6 +268,8 @@ visual observations is 84 by 84 (matches the parameter found in `dqn_agent.py` and `rainbow_agent.py`). Dopamine's agents currently do not automatically adapt to the observation dimensions or number of channels. +Dopamine only works for single agent environments. You will need to create an environment with a single agent for Dopamine to work. + ### Hyperparameters The hyperparameters provided by Dopamine are tailored to the Atari games, and From 9e58ca85b3a2118304f17e6972ad63bce678715d Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 19:02:11 -0700 Subject: [PATCH 17/19] Adding some docstrings --- gym-unity/gym_unity/envs/gym_env.py | 57 +++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py index 91b0c41946e..d7b90158f3c 100644 --- a/gym-unity/gym_unity/envs/gym_env.py +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -21,6 +21,25 @@ def _parse_behavior(full_behavior): class AgentStatus: + """ + Some information about an Agent. + - behavior: The name of the behavior the agent has + - team: The team of the agent. (useful for self play). Agents in different teams + face each other. + - group: The cooperative group the agent belongs to. + - group_reward: The reward the group the agent belongs to received since the last + action of the agent + - action_mask: For discrete and multi-discrete action spaces only. A True mask + means that an action is not available. + - interrupted: If True and the agent is done, it means that the agent entered + the done state not because of its behavior but because of a task independent + event. + - current_obs: The current observation of the agent. + - reward : The reward the agent received since the last action + - done: Whether the agent's task has ended + - needs_action: if True, the agent needs an action to proceed. + """ + def __init__( self, behavior, @@ -48,6 +67,18 @@ def __init__( class UnityToGymWrapper(gym.Env): def __init__(self, env: BaseEnv, action_space_seed: Optional[int] = None): + """ + A multi-agent gym wrapper for Unity environments. Note that in the single + agent case, the environment is equivalent to a typical single agent gym + environment. When there is more than one agent in the environment, use the + `env.active` property to access information about the active agent. + Note that when calling step, the active agent will change, you will need + to keep track of the action that was performed by each agent since the + observation received from a call to step will not necessarily be from the + agent that just received the action. + :param env: The UnityEnvironment that is being wrapped. + :param action_space_seed: The seed for the action spaces of the agents. + """ atexit.register(self.close) self._env = env self._agent_index = -1 @@ -85,6 +116,9 @@ def active(self) -> Optional[AgentStatus]: @property def observation_space(self) -> Optional[spaces.Space]: + """ + The observation space of the active agent. + """ self._assert_loaded() current_behavior = self._current_behavior_name() if current_behavior is None: @@ -104,6 +138,9 @@ def observation_space(self) -> Optional[spaces.Space]: @property def action_space(self) -> Optional[spaces.Space]: + """ + The action space of the active agent. + """ self._assert_loaded() current_behavior = self._current_behavior_name() if current_behavior is None: @@ -139,17 +176,30 @@ def action_space(self) -> Optional[spaces.Space]: @property def reward_range(self) -> Tuple[float, float]: + """ + The reward range of the active agent. + """ self._assert_loaded() return -float("inf"), float("inf") @property def side_channel(self) -> Dict[str, Any]: + """ + The side channels of the environment. You can access the side channels + of an environment with `env.side_channel[]`. + """ self._assert_loaded() return self._side_channel_dict def step( self, action: Any = None ) -> Tuple[List[np.array], float, bool, Optional[Dict[str, Any]]]: + """ + Sets the action of the active agent and get the observation, reward, done + and info of the next agent. + :param action: The action for the active agent + :returns: Observation, reward, done and info for the next agent. + """ self._assert_loaded() # Convert Actions if action is not None: @@ -260,6 +310,10 @@ def step( return self.step() def reset(self) -> List[np.array]: + """ + Resets the environment. + :return: The observation of a first agent + """ self._assert_loaded() self._env.reset() self._agent_index = -1 @@ -270,6 +324,9 @@ def reset(self) -> List[np.array]: return obs def close(self) -> None: + """ + Close the environment. + """ if self._env is not None: self._env.close() self._env = None From 5163f09ae66c19e67e026ff563f5af27811bad8e Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 27 Jul 2021 19:12:26 -0700 Subject: [PATCH 18/19] adding a comment around the hack --- gym-unity/gym_unity/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gym-unity/gym_unity/__init__.py b/gym-unity/gym_unity/__init__.py index fb995500a8d..08baaaa6419 100644 --- a/gym-unity/gym_unity/__init__.py +++ b/gym-unity/gym_unity/__init__.py @@ -7,4 +7,7 @@ try: import gym_unity.envs # noqa except ImportError: + # Try here because when calling setup, we access __version__ but do not have + # gym installed yet. This is to make installation not raise an error. + # we want to be able to use the gym registry after calling `import gym_unity` pass From 5c9eddb744478a09407ad1a8bb4174c9c0911a41 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Wed, 4 Aug 2021 10:31:56 -0700 Subject: [PATCH 19/19] adding min gym version because of spaces.Space --- gym-unity/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gym-unity/setup.py b/gym-unity/setup.py index b234ed3bc59..fb6f81b8cb6 100755 --- a/gym-unity/setup.py +++ b/gym-unity/setup.py @@ -38,6 +38,6 @@ def run(self): author_email="ML-Agents@unity3d.com", url="https://github.com/Unity-Technologies/ml-agents", packages=find_packages(), - install_requires=["gym", f"mlagents_envs=={VERSION}", "certifi"], + install_requires=["gym>=0.11.0", f"mlagents_envs=={VERSION}", "certifi"], cmdclass={"verify": VerifyVersionCommand}, )