diff --git a/gym-unity/README.md b/gym-unity/README.md index f664c16e467..21ef03e39f4 100755 --- a/gym-unity/README.md +++ b/gym-unity/README.md @@ -31,42 +31,34 @@ from the root of the project repository use: ```python from gym_unity.envs import UnityToGymWrapper -env = UnityToGymWrapper(unity_env, uint8_visual, flatten_branched, allow_multiple_obs) +env = UnityToGymWrapper(unity_env) ``` - `unity_env` refers to the Unity environment to be wrapped. -- `uint8_visual` refers to whether to output visual observations as `uint8` - values (0-255). Many common Gym environments (e.g. Atari) do this. By default - they will be floats (0.0-1.0). Defaults to `False`. +- `action_space_seed` is the optional seed for action sampling. If non-None, will + be used to set the random seed on created gym.Space instances. -- `flatten_branched` will flatten a branched discrete action space into a Gym - Discrete. Otherwise, it will be converted into a MultiDiscrete. Defaults to - `False`. +The returned environment `env` will function as a multi-agent gym. -- `allow_multiple_obs` will return a list of observations. The first elements - contain the visual observations and the last element contains the array of - vector observations. If False the environment returns a single array (containing - a single visual observations, if present, otherwise the vector observation). - Defaults to `False`. +Alternatively, you can get one of the environments from the Unity registry with the following code : -- `action_space_seed` is the optional seed for action sampling. If non-None, will - be used to set the random seed on created gym.Space instances. +``` +import gym +import gym_unity -The returned environment `env` will function as a gym. +env = gym.make("GridWorld-v0") +... +env.close() + +``` + +> **:warning: IMPORTANT** +> Note that a Unity environment can have **ANY** number of different agents all requesting decisions at different and irregular times. When calling `env.step()` the next observation returned can be for a completely different agent than the one the action was for. +To know which agent is currently in need of a decision, you can access the current agent identifier by calling `env.active.agent`. Calling `env.step()` will not necessarily move the simulation forward: `env.step()` will set the action for the `env.active.agent`. When all agents that needed an action received one, the simulation will move forward. ## Limitations -- It is only possible to use an environment with a **single** Agent. -- By default, the first visual observation is provided as the `observation`, if - present. Otherwise, vector observations are provided. You can receive all - visual and vector observations by using the `allow_multiple_obs=True` option in - the gym parameters. If set to `True`, you will receive a list of `observation` - instead of only one. -- The `TerminalSteps` or `DecisionSteps` output from the environment can still - be accessed from the `info` provided by `env.step(action)`. -- Stacked vector observations are not supported. -- Environment registration for use with `gym.make()` is currently not supported. - Calling env.render() will not render a new frame of the environment. It will return the latest visual observation if using visual observations. @@ -80,6 +72,8 @@ using these algorithms. This requires the creation of custom training scripts to launch each algorithm. In most cases these scripts can be created by making slight modifications to the ones provided for Atari and Mujoco environments. +Note that Baselines assumes that there is a single agent in the environment. Baselines will not work on environments where the number of agents is not strictly one. + These examples were tested with baselines version 0.1.6. ### Example - DQN Baseline @@ -274,6 +268,8 @@ visual observations is 84 by 84 (matches the parameter found in `dqn_agent.py` and `rainbow_agent.py`). Dopamine's agents currently do not automatically adapt to the observation dimensions or number of channels. +Dopamine only works for single agent environments. You will need to create an environment with a single agent for Dopamine to work. + ### Hyperparameters The hyperparameters provided by Dopamine are tailored to the Atari games, and diff --git a/gym-unity/gym_unity/__init__.py b/gym-unity/gym_unity/__init__.py index 7902085efe7..08baaaa6419 100644 --- a/gym-unity/gym_unity/__init__.py +++ b/gym-unity/gym_unity/__init__.py @@ -3,3 +3,11 @@ # Git tag that will be checked to determine whether to trigger upload to pypi __release_tag__ = None + +try: + import gym_unity.envs # noqa +except ImportError: + # Try here because when calling setup, we access __version__ but do not have + # gym installed yet. This is to make installation not raise an error. + # we want to be able to use the gym registry after calling `import gym_unity` + pass diff --git a/gym-unity/gym_unity/envs/__init__.py b/gym-unity/gym_unity/envs/__init__.py index 322d83659bb..2ba5b99a6f0 100644 --- a/gym-unity/gym_unity/envs/__init__.py +++ b/gym-unity/gym_unity/envs/__init__.py @@ -1,361 +1,47 @@ -import itertools -import numpy as np -from typing import Any, Dict, List, Optional, Tuple, Union - -import gym -from gym import error, spaces - -from mlagents_envs.base_env import ActionTuple, BaseEnv -from mlagents_envs.base_env import DecisionSteps, TerminalSteps -from mlagents_envs import logging_util - - -class UnityGymException(error.Error): - """ - Any error related to the gym wrapper of ml-agents. - """ - - pass - - -logger = logging_util.get_logger(__name__) -logging_util.set_log_level(logging_util.INFO) - -GymStepResult = Tuple[np.ndarray, float, bool, Dict] - - -class UnityToGymWrapper(gym.Env): - """ - Provides Gym wrapper for Unity Learning Environments. - """ - - def __init__( - self, - unity_env: BaseEnv, - uint8_visual: bool = False, - flatten_branched: bool = False, - allow_multiple_obs: bool = False, - action_space_seed: Optional[int] = None, - ): - """ - Environment initialization - :param unity_env: The Unity BaseEnv to be wrapped in the gym. Will be closed when the UnityToGymWrapper closes. - :param uint8_visual: Return visual observations as uint8 (0-255) matrices instead of float (0.0-1.0). - :param flatten_branched: If True, turn branched discrete action spaces into a Discrete space rather than - MultiDiscrete. - :param allow_multiple_obs: If True, return a list of np.ndarrays as observations with the first elements - containing the visual observations and the last element containing the array of vector observations. - If False, returns a single np.ndarray containing either only a single visual observation or the array of - vector observations. - :param action_space_seed: If non-None, will be used to set the random seed on created gym.Space instances. - """ - self._env = unity_env - - # Take a single step so that the brain information will be sent over - if not self._env.behavior_specs: - self._env.step() - - self.visual_obs = None - - # Save the step result from the last time all Agents requested decisions. - self._previous_decision_step: DecisionSteps = None - self._flattener = None - # Hidden flag used by Atari environments to determine if the game is over - self.game_over = False - self._allow_multiple_obs = allow_multiple_obs - - # Check brain configuration - if len(self._env.behavior_specs) != 1: - raise UnityGymException( - "There can only be one behavior in a UnityEnvironment " - "if it is wrapped in a gym." - ) - - self.name = list(self._env.behavior_specs.keys())[0] - self.group_spec = self._env.behavior_specs[self.name] - - if self._get_n_vis_obs() == 0 and self._get_vec_obs_size() == 0: - raise UnityGymException( - "There are no observations provided by the environment." - ) - - if not self._get_n_vis_obs() >= 1 and uint8_visual: - logger.warning( - "uint8_visual was set to true, but visual observations are not in use. " - "This setting will not have any effect." - ) - else: - self.uint8_visual = uint8_visual - if ( - self._get_n_vis_obs() + self._get_vec_obs_size() >= 2 - and not self._allow_multiple_obs - ): - logger.warning( - "The environment contains multiple observations. " - "You must define allow_multiple_obs=True to receive them all. " - "Otherwise, only the first visual observation (or vector observation if" - "there are no visual observations) will be provided in the observation." - ) - - # Check for number of agents in scene. - self._env.reset() - decision_steps, _ = self._env.get_steps(self.name) - self._check_agents(len(decision_steps)) - self._previous_decision_step = decision_steps - - # Set action spaces - if self.group_spec.action_spec.is_discrete(): - self.action_size = self.group_spec.action_spec.discrete_size - branches = self.group_spec.action_spec.discrete_branches - if self.group_spec.action_spec.discrete_size == 1: - self._action_space = spaces.Discrete(branches[0]) - else: - if flatten_branched: - self._flattener = ActionFlattener(branches) - self._action_space = self._flattener.action_space - else: - self._action_space = spaces.MultiDiscrete(branches) - - elif self.group_spec.action_spec.is_continuous(): - if flatten_branched: - logger.warning( - "The environment has a non-discrete action space. It will " - "not be flattened." - ) - - self.action_size = self.group_spec.action_spec.continuous_size - high = np.array([1] * self.group_spec.action_spec.continuous_size) - self._action_space = spaces.Box(-high, high, dtype=np.float32) - else: - raise UnityGymException( - "The gym wrapper does not provide explicit support for both discrete " - "and continuous actions." - ) - - if action_space_seed is not None: - self._action_space.seed(action_space_seed) - - # Set observations space - list_spaces: List[gym.Space] = [] - shapes = self._get_vis_obs_shape() - for shape in shapes: - if uint8_visual: - list_spaces.append(spaces.Box(0, 255, dtype=np.uint8, shape=shape)) - else: - list_spaces.append(spaces.Box(0, 1, dtype=np.float32, shape=shape)) - if self._get_vec_obs_size() > 0: - # vector observation is last - high = np.array([np.inf] * self._get_vec_obs_size()) - list_spaces.append(spaces.Box(-high, high, dtype=np.float32)) - if self._allow_multiple_obs: - self._observation_space = spaces.Tuple(list_spaces) - else: - self._observation_space = list_spaces[0] # only return the first one - - def reset(self) -> Union[List[np.ndarray], np.ndarray]: - """Resets the state of the environment and returns an initial observation. - Returns: observation (object/list): the initial observation of the - space. - """ - self._env.reset() - decision_step, _ = self._env.get_steps(self.name) - n_agents = len(decision_step) - self._check_agents(n_agents) - self.game_over = False - - res: GymStepResult = self._single_step(decision_step) - return res[0] - - def step(self, action: List[Any]) -> GymStepResult: - """Run one timestep of the environment's dynamics. When end of - episode is reached, you are responsible for calling `reset()` - to reset this environment's state. - Accepts an action and returns a tuple (observation, reward, done, info). - Args: - action (object/list): an action provided by the environment - Returns: - observation (object/list): agent's observation of the current environment - reward (float/list) : amount of reward returned after previous action - done (boolean/list): whether the episode has ended. - info (dict): contains auxiliary diagnostic information. - """ - if self.game_over: - raise UnityGymException( - "You are calling 'step()' even though this environment has already " - "returned done = True. You must always call 'reset()' once you " - "receive 'done = True'." - ) - if self._flattener is not None: - # Translate action into list - action = self._flattener.lookup_action(action) - - action = np.array(action).reshape((1, self.action_size)) - - action_tuple = ActionTuple() - if self.group_spec.action_spec.is_continuous(): - action_tuple.add_continuous(action) - else: - action_tuple.add_discrete(action) - self._env.set_actions(self.name, action_tuple) - - self._env.step() - decision_step, terminal_step = self._env.get_steps(self.name) - self._check_agents(max(len(decision_step), len(terminal_step))) - if len(terminal_step) != 0: - # The agent is done - self.game_over = True - return self._single_step(terminal_step) - else: - return self._single_step(decision_step) - - def _single_step(self, info: Union[DecisionSteps, TerminalSteps]) -> GymStepResult: - if self._allow_multiple_obs: - visual_obs = self._get_vis_obs_list(info) - visual_obs_list = [] - for obs in visual_obs: - visual_obs_list.append(self._preprocess_single(obs[0])) - default_observation = visual_obs_list - if self._get_vec_obs_size() >= 1: - default_observation.append(self._get_vector_obs(info)[0, :]) +from gym.envs.registration import register +from gym.error import Error +from mlagents_envs.registry import default_registry +from gym_unity.envs.gym_env import UnityToGymWrapper + + +from mlagents_envs.side_channel.engine_configuration_channel import ( + EngineConfigurationChannel, +) +from mlagents_envs.side_channel.stats_side_channel import StatsSideChannel +from mlagents_envs.side_channel.environment_parameters_channel import ( + EnvironmentParametersChannel, +) +from mlagents_envs.exception import UnityWorkerInUseException + + +def gym_entry_point(env_name): + def create(action_space_seed=None, **kwargs): + if "side_channels" not in kwargs: + kwargs["side_channels"] = [ + EngineConfigurationChannel(), + EnvironmentParametersChannel(), + StatsSideChannel(), + ] + _e = None + if "base_port" not in kwargs: + port = 6000 + while _e is None: + try: + kwargs["base_port"] = port + _e = default_registry[env_name].make(**kwargs) + except UnityWorkerInUseException: + port += 1 + pass else: - if self._get_n_vis_obs() >= 1: - visual_obs = self._get_vis_obs_list(info) - default_observation = self._preprocess_single(visual_obs[0][0]) - else: - default_observation = self._get_vector_obs(info)[0, :] - - if self._get_n_vis_obs() >= 1: - visual_obs = self._get_vis_obs_list(info) - self.visual_obs = self._preprocess_single(visual_obs[0][0]) - - done = isinstance(info, TerminalSteps) - - return (default_observation, info.reward[0], done, {"step": info}) - - def _preprocess_single(self, single_visual_obs: np.ndarray) -> np.ndarray: - if self.uint8_visual: - return (255.0 * single_visual_obs).astype(np.uint8) - else: - return single_visual_obs - - def _get_n_vis_obs(self) -> int: - result = 0 - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 3: - result += 1 - return result - - def _get_vis_obs_shape(self) -> List[Tuple]: - result: List[Tuple] = [] - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 3: - result.append(obs_spec.shape) - return result - - def _get_vis_obs_list( - self, step_result: Union[DecisionSteps, TerminalSteps] - ) -> List[np.ndarray]: - result: List[np.ndarray] = [] - for obs in step_result.obs: - if len(obs.shape) == 4: - result.append(obs) - return result - - def _get_vector_obs( - self, step_result: Union[DecisionSteps, TerminalSteps] - ) -> np.ndarray: - result: List[np.ndarray] = [] - for obs in step_result.obs: - if len(obs.shape) == 2: - result.append(obs) - return np.concatenate(result, axis=1) - - def _get_vec_obs_size(self) -> int: - result = 0 - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 1: - result += obs_spec.shape[0] - return result - - def render(self, mode="rgb_array"): - """ - Return the latest visual observations. - Note that it will not render a new frame of the environment. - """ - return self.visual_obs - - def close(self) -> None: - """Override _close in your subclass to perform any necessary cleanup. - Environments will automatically close() themselves when - garbage collected or when the program exits. - """ - self._env.close() - - def seed(self, seed: Any = None) -> None: - """Sets the seed for this env's random number generator(s). - Currently not implemented. - """ - logger.warning("Could not seed environment %s", self.name) - return - - @staticmethod - def _check_agents(n_agents: int) -> None: - if n_agents > 1: - raise UnityGymException( - f"There can only be one Agent in the environment but {n_agents} were detected." - ) - - @property - def metadata(self): - return {"render.modes": ["rgb_array"]} - - @property - def reward_range(self) -> Tuple[float, float]: - return -float("inf"), float("inf") - - @property - def action_space(self) -> gym.Space: - return self._action_space - - @property - def observation_space(self): - return self._observation_space - - -class ActionFlattener: - """ - Flattens branched discrete action spaces into single-branch discrete action spaces. - """ + _e = default_registry[env_name].make(**kwargs) + return UnityToGymWrapper(_e, action_space_seed) - def __init__(self, branched_action_space): - """ - Initialize the flattener. - :param branched_action_space: A List containing the sizes of each branch of the action - space, e.g. [2,3,3] for three branches with size 2, 3, and 3 respectively. - """ - self._action_shape = branched_action_space - self.action_lookup = self._create_lookup(self._action_shape) - self.action_space = spaces.Discrete(len(self.action_lookup)) + return create - @classmethod - def _create_lookup(self, branched_action_space): - """ - Creates a Dict that maps discrete actions (scalars) to branched actions (lists). - Each key in the Dict maps to one unique set of branched actions, and each value - contains the List of branched actions. - """ - possible_vals = [range(_num) for _num in branched_action_space] - all_actions = [list(_action) for _action in itertools.product(*possible_vals)] - # Dict should be faster than List for large action spaces - action_lookup = { - _scalar: _action for (_scalar, _action) in enumerate(all_actions) - } - return action_lookup - def lookup_action(self, action): - """ - Convert a scalar discrete action into a unique set of branched actions. - :param: action: A scalar value representing one of the discrete actions. - :return: The List containing the branched actions. - """ - return self.action_lookup[action] +for key in default_registry: + registry_key = key + "-v0" + try: + register(id=registry_key, entry_point=gym_entry_point(key)) + except Error: + pass diff --git a/gym-unity/gym_unity/envs/gym_env.py b/gym-unity/gym_unity/envs/gym_env.py new file mode 100644 index 00000000000..d7b90158f3c --- /dev/null +++ b/gym-unity/gym_unity/envs/gym_env.py @@ -0,0 +1,335 @@ +import numpy as np +from typing import Any, Dict, List, Optional, Tuple + +import atexit +import gym +from gym import error, spaces +from urllib.parse import urlparse, parse_qs + +from mlagents_envs.base_env import ActionTuple, BaseEnv +from mlagents_envs.base_env import DecisionSteps, TerminalSteps + + +def _parse_behavior(full_behavior): + parsed = urlparse(full_behavior) + name = parsed.path + ids = parse_qs(parsed.query) + team_id: int = 0 + if "team" in ids: + team_id = int(ids["team"][0]) + return name, team_id + + +class AgentStatus: + """ + Some information about an Agent. + - behavior: The name of the behavior the agent has + - team: The team of the agent. (useful for self play). Agents in different teams + face each other. + - group: The cooperative group the agent belongs to. + - group_reward: The reward the group the agent belongs to received since the last + action of the agent + - action_mask: For discrete and multi-discrete action spaces only. A True mask + means that an action is not available. + - interrupted: If True and the agent is done, it means that the agent entered + the done state not because of its behavior but because of a task independent + event. + - current_obs: The current observation of the agent. + - reward : The reward the agent received since the last action + - done: Whether the agent's task has ended + - needs_action: if True, the agent needs an action to proceed. + """ + + def __init__( + self, + behavior, + agent, + group, + group_reward, + action_mask, + interrupted, + current_obs, + reward, + done, + needs_action, + ): + self.behavior, self.team = _parse_behavior(behavior) + self.agent = agent + self.group = group + self.group_reward = group_reward + self.action_mask = action_mask + self.interrupted = interrupted + self.obs = current_obs + self.reward = reward + self.done = done + self.needs_action = needs_action + + +class UnityToGymWrapper(gym.Env): + def __init__(self, env: BaseEnv, action_space_seed: Optional[int] = None): + """ + A multi-agent gym wrapper for Unity environments. Note that in the single + agent case, the environment is equivalent to a typical single agent gym + environment. When there is more than one agent in the environment, use the + `env.active` property to access information about the active agent. + Note that when calling step, the active agent will change, you will need + to keep track of the action that was performed by each agent since the + observation received from a call to step will not necessarily be from the + agent that just received the action. + :param env: The UnityEnvironment that is being wrapped. + :param action_space_seed: The seed for the action spaces of the agents. + """ + atexit.register(self.close) + self._env = env + self._agent_index = -1 + self._current_batch: Optional[Tuple[DecisionSteps, TerminalSteps]] = None + self._behavior_index = 0 + self._current_action: Optional[ActionTuple] = None + self._last: Optional[AgentStatus] = None + self._action_spaces: Dict[str, spaces.Space] = {} + self._obs_spaces: Dict[str, spaces.Space] = {} + self._action_sampling_seed = action_space_seed + self._side_channel_dict = { + type(v).__name__: v + for v in self._env._side_channel_manager._side_channels_dict.values() + } + + def _current_behavior_name(self) -> Optional[str]: + names = list(self._env.behavior_specs.keys()) + if len(names) > self._behavior_index: + return names[self._behavior_index] + return None + + def _assert_loaded(self) -> None: + if self._env is None: + raise error.Error("No environment loaded") + + @property + def active(self) -> Optional[AgentStatus]: + """ + This is not part of the original gym api. It is needed to know which behaviors + and agent ID the observation from a reset call belongs to. + """ + # TODO : info won't do. Info is supposed to be "extra" information, not needed + self._assert_loaded() + return self._last + + @property + def observation_space(self) -> Optional[spaces.Space]: + """ + The observation space of the active agent. + """ + self._assert_loaded() + current_behavior = self._current_behavior_name() + if current_behavior is None: + return None + if current_behavior in self._obs_spaces: + return self._obs_spaces[current_behavior] + obs_spec = self._env.behavior_specs[current_behavior].observation_specs + obs_spaces = tuple( + spaces.Box(-np.inf, np.inf, shape=spec.shape, dtype=np.float32) + for spec in obs_spec + ) + if len(obs_spaces) == 1: + self._obs_spaces[current_behavior] = obs_spaces[0] + else: + self._obs_spaces[current_behavior] = spaces.Tuple(obs_spaces) + return self._obs_spaces[current_behavior] + + @property + def action_space(self) -> Optional[spaces.Space]: + """ + The action space of the active agent. + """ + self._assert_loaded() + current_behavior = self._current_behavior_name() + if current_behavior is None: + return None + if current_behavior in self._action_spaces: + return self._action_spaces[current_behavior] + act_spec = self._env.behavior_specs[current_behavior].action_spec + if act_spec.continuous_size == 0 and len(act_spec.discrete_branches) == 0: + raise error.Error("No actions found") + if act_spec.discrete_size == 1: + d_space = spaces.Discrete(act_spec.discrete_branches[0]) + if self._action_sampling_seed is not None: + d_space.seed(self._action_sampling_seed) + if act_spec.continuous_size == 0: + self._action_spaces[current_behavior] = d_space + return self._action_spaces[current_behavior] + if act_spec.discrete_size > 0: + d_space = spaces.MultiDiscrete(act_spec.discrete_branches) + if self._action_sampling_seed is not None: + d_space.seed(self._action_sampling_seed) + if act_spec.continuous_size == 0: + self._action_spaces[current_behavior] = d_space + return self._action_spaces[current_behavior] + if act_spec.continuous_size > 0: + c_space = spaces.Box(-1, 1, (act_spec.continuous_size,), dtype=np.int32) + if self._action_sampling_seed is not None: + c_space.seed(self._action_sampling_seed) + if len(act_spec.discrete_branches) == 0: + self._action_spaces[current_behavior] = c_space + return self._action_spaces[current_behavior] + self._action_spaces[current_behavior] = spaces.Tuple((c_space, d_space)) + return self._action_spaces[current_behavior] + + @property + def reward_range(self) -> Tuple[float, float]: + """ + The reward range of the active agent. + """ + self._assert_loaded() + return -float("inf"), float("inf") + + @property + def side_channel(self) -> Dict[str, Any]: + """ + The side channels of the environment. You can access the side channels + of an environment with `env.side_channel[]`. + """ + self._assert_loaded() + return self._side_channel_dict + + def step( + self, action: Any = None + ) -> Tuple[List[np.array], float, bool, Optional[Dict[str, Any]]]: + """ + Sets the action of the active agent and get the observation, reward, done + and info of the next agent. + :param action: The action for the active agent + :returns: Observation, reward, done and info for the next agent. + """ + self._assert_loaded() + # Convert Actions + if action is not None: + if not self.action_space.contains(action): # type: ignore + raise error.Error( + f"Invalid action, got {action} but was expecting action from {self.action_space}" + ) + if isinstance(self.action_space, spaces.Tuple): + action = ActionTuple(action[0], action[1]) + elif isinstance(self.action_space, spaces.MultiDiscrete): + action = ActionTuple(None, action) + elif isinstance(self.action_space, spaces.Discrete): + action = ActionTuple(None, np.array(action).reshape(1, 1)) + else: + action = ActionTuple(action, None) + + if self._current_batch is None: + raise error.Error( + "You must reset the environment before you can perform a step" + ) + + decision_batch, termination_batch = self._current_batch + + # Build action + if self._current_action is None: + a_spec = self._env.behavior_specs[self._current_behavior_name()].action_spec + self._current_action = ActionTuple( + np.zeros( + (len(decision_batch), a_spec.continuous_size), dtype=np.float32 + ), + np.zeros( + (len(decision_batch), len(a_spec.discrete_branches)), dtype=np.int32 + ), + ) + if action is not None: + if self._agent_index >= 0 and self._agent_index < len(decision_batch): + + if action.continuous is not None: + self._current_action.continuous[ + self._agent_index + ] = action.continuous[0] + if action.discrete is not None: + self._current_action.discrete[self._agent_index] = action.discrete[ + 0 + ] + else: + # A Useless action was passed (the last.agent was done) + pass + + self._agent_index += 1 + + if self._agent_index < len(termination_batch): + # The index is within the terminal steps + index = self._agent_index + obs = [batch_obs[index] for batch_obs in termination_batch.obs] + obs = obs if len(obs) > 1 else obs[0] + reward = termination_batch.reward[index] + done = True + self._last = AgentStatus( + list(self._env.behavior_specs.keys())[self._behavior_index], + termination_batch.agent_id[index], + termination_batch.group_id[index], + termination_batch.group_reward[index], + None, + termination_batch.interrupted[index], + obs, + reward, + done, + False, + ) + return obs, reward, done, None + + if self._agent_index < len(decision_batch) + len(termination_batch): + index = self._agent_index - len(termination_batch) + # the index is within the decsion steps + obs = [batch_obs[index] for batch_obs in decision_batch.obs] + obs = obs if len(obs) > 1 else obs[0] + reward = decision_batch.reward[index] + done = False + + self._last = AgentStatus( + list(self._env.behavior_specs.keys())[self._behavior_index], + decision_batch.agent_id[index], + decision_batch.group_id[index], + decision_batch.group_reward[index], + [mask[index] for mask in decision_batch.action_mask] + if decision_batch.action_mask is not None + else None, + False, + obs, + reward, + done, + True, + ) + return obs, reward, done, None + + # The index is too high, time to set the action for the agents we have + if self._current_action is not None: + self._env.set_actions(self._current_behavior_name(), self._current_action) + self._current_action = None + self._behavior_index += 1 + if self._behavior_index >= len(self._env.behavior_specs): + self._env.step() + self._behavior_index = 0 + self._current_batch = self._env.get_steps(self._current_behavior_name()) + self._agent_index = -1 + + return self.step() + + def reset(self) -> List[np.array]: + """ + Resets the environment. + :return: The observation of a first agent + """ + self._assert_loaded() + self._env.reset() + self._agent_index = -1 + self._behavior_index = 0 + self._current_batch = self._env.get_steps(self._current_behavior_name()) + self._current_action = None + obs, _, _, _ = self.step() + return obs + + def close(self) -> None: + """ + Close the environment. + """ + if self._env is not None: + self._env.close() + self._env = None + + def __del__(self) -> None: + self.close() diff --git a/gym-unity/gym_unity/tests/test_gym.py b/gym-unity/gym_unity/tests/test_gym.py index 6928faa0fd2..df1f3c6588e 100644 --- a/gym-unity/gym_unity/tests/test_gym.py +++ b/gym-unity/gym_unity/tests/test_gym.py @@ -1,277 +1,33 @@ -from unittest import mock import pytest -import numpy as np -from gym import spaces -from gym_unity.envs import UnityToGymWrapper -from mlagents_envs.base_env import ( - BehaviorSpec, - ActionSpec, - DecisionSteps, - TerminalSteps, - BehaviorMapping, -) -from mlagents.trainers.tests.dummy_config import create_observation_specs_with_shapes +import gym +import gym_unity # noqa -def test_gym_wrapper(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec() - mock_decision_step, mock_terminal_step = create_mock_vector_steps(mock_spec) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - env = UnityToGymWrapper(mock_env) - assert isinstance(env.reset(), np.ndarray) - actions = env.action_space.sample() - assert actions.shape[0] == 2 - obs, rew, done, info = env.step(actions) +@pytest.mark.parametrize("env_name", ["3DBall-v0", "WallJump-v0", "PushBlock-v0"]) +def test_env(env_name): + env = gym.make(env_name, no_graphics=True, base_port=6000) + obs = env.reset() assert env.observation_space.contains(obs) - assert isinstance(obs, np.ndarray) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - -def test_branched_flatten(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - vector_action_space_type="discrete", vector_action_space_size=[2, 2, 3] - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, num_agents=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, flatten_branched=True) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 12 - assert env._flattener.lookup_action(0) == [0, 0, 0] - assert env._flattener.lookup_action(11) == [1, 1, 2] - - # Check that False produces a MultiDiscrete - env = UnityToGymWrapper(mock_env, flatten_branched=False) - assert isinstance(env.action_space, spaces.MultiDiscrete) - - -def test_action_space(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - vector_action_space_type="discrete", vector_action_space_size=[5] - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, num_agents=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, flatten_branched=True) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 5 - - env = UnityToGymWrapper(mock_env, flatten_branched=False) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 5 - - -def test_action_space_seed(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec() - mock_decision_step, mock_terminal_step = create_mock_vector_steps(mock_spec) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - actions = [] - for _ in range(0, 2): - env = UnityToGymWrapper(mock_env, action_space_seed=1337) - env.reset() - actions.append(env.action_space.sample()) - assert (actions[0] == actions[1]).all() - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_visual(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=1, vector_observation_space_size=0 - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8) - assert isinstance(env.observation_space, spaces.Box) - assert isinstance(env.reset(), np.ndarray) - actions = env.action_space.sample() - assert actions.shape[0] == 2 - obs, rew, done, info = env.step(actions) + for _ in range(10): + obs, _, _, _ = env.step(env.action_space.sample()) + assert env.observation_space.contains(obs) + obs = env.reset() assert env.observation_space.contains(obs) - assert isinstance(obs, np.ndarray) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_single_visual_and_vector(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=1, - vector_observation_space_size=3, - vector_action_space_size=[2], - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=True) - assert isinstance(env.observation_space, spaces.Tuple) - assert len(env.observation_space) == 2 - reset_obs = env.reset() - assert isinstance(reset_obs, list) - assert len(reset_obs) == 2 - assert all(isinstance(ob, np.ndarray) for ob in reset_obs) - assert reset_obs[-1].shape == (3,) - assert len(reset_obs[0].shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, list) - assert len(obs) == 2 - assert all(isinstance(ob, np.ndarray) for ob in obs) - assert reset_obs[-1].shape == (3,) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - # check behavior for allow_multiple_obs = False - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=False) - assert isinstance(env.observation_space, spaces.Box) - reset_obs = env.reset() - assert isinstance(reset_obs, np.ndarray) - assert len(reset_obs.shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, np.ndarray) - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_multi_visual_and_vector(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=2, - vector_observation_space_size=3, - vector_action_space_size=[2], - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=2 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=True) - assert isinstance(env.observation_space, spaces.Tuple) - assert len(env.observation_space) == 3 - reset_obs = env.reset() - assert isinstance(reset_obs, list) - assert len(reset_obs) == 3 - assert all(isinstance(ob, np.ndarray) for ob in reset_obs) - assert reset_obs[-1].shape == (3,) - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert all(isinstance(ob, np.ndarray) for ob in obs) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - # check behavior for allow_multiple_obs = False - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=False) - assert isinstance(env.observation_space, spaces.Box) - reset_obs = env.reset() - assert isinstance(reset_obs, np.ndarray) - assert len(reset_obs.shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, np.ndarray) - - -# Helper methods - - -def create_mock_group_spec( - number_visual_observations=0, - vector_action_space_type="continuous", - vector_observation_space_size=3, - vector_action_space_size=None, -): - """ - Creates a mock BrainParameters object with parameters. - """ - # Avoid using mutable object as default param - if vector_action_space_type == "continuous": - if vector_action_space_size is None: - vector_action_space_size = 2 - else: - vector_action_space_size = vector_action_space_size[0] - action_spec = ActionSpec.create_continuous(vector_action_space_size) - else: - if vector_action_space_size is None: - vector_action_space_size = (2,) - else: - vector_action_space_size = tuple(vector_action_space_size) - action_spec = ActionSpec.create_discrete(vector_action_space_size) - obs_shapes = [(vector_observation_space_size,)] - for _ in range(number_visual_observations): - obs_shapes += [(8, 8, 3)] - obs_spec = create_observation_specs_with_shapes(obs_shapes) - return BehaviorSpec(obs_spec, action_spec) - - -def create_mock_vector_steps(specs, num_agents=1, number_visual_observations=0): - """ - Creates a mock BatchedStepResult with vector observations. Imitates constant - vector observations, rewards, dones, and agents. - - :BehaviorSpecs specs: The BehaviorSpecs for this mock - :int num_agents: Number of "agents" to imitate in your BatchedStepResult values. - """ - obs = [np.array([num_agents * [1, 2, 3]]).reshape(num_agents, 3)] - if number_visual_observations: - obs += [ - np.zeros(shape=(num_agents, 8, 8, 3), dtype=np.float32) - ] * number_visual_observations - rewards = np.array(num_agents * [1.0]) - agents = np.array(range(0, num_agents)) - group_id = np.array(num_agents * [0]) - group_rewards = np.array(num_agents * [0.0]) - return ( - DecisionSteps(obs, rewards, agents, None, group_id, group_rewards), - TerminalSteps.empty(specs), - ) - - -def setup_mock_unityenvironment(mock_env, mock_spec, mock_decision, mock_termination): - """ - Takes a mock UnityEnvironment and adds the appropriate properties, defined by the mock - GroupSpec and BatchedStepResult. - - :Mock mock_env: A mock UnityEnvironment, usually empty. - :Mock mock_spec: An AgentGroupSpec object that specifies the params of this environment. - :Mock mock_decision: A DecisionSteps object that will be returned at each step and reset. - :Mock mock_termination: A TerminationSteps object that will be returned at each step and reset. - """ - mock_env.behavior_specs = BehaviorMapping({"MockBrain": mock_spec}) - mock_env.get_steps.return_value = (mock_decision, mock_termination) + for _ in range(10): + obs, _, _, _ = env.step(env.action_space.sample()) + assert env.observation_space.contains(obs) + assert isinstance(env.action_space, gym.spaces.Space) + assert isinstance(env.observation_space, gym.spaces.Space) + env.close() + + env = gym.make(env_name, no_graphics=True, action_space_seed=42) + env.reset() + rng_1 = env.action_space._np_random + env.close() + + env = gym.make(env_name, no_graphics=True, action_space_seed=42) + env.reset() + rng_2 = env.action_space._np_random + env.close() + assert rng_1.randint(10000000000000) == rng_2.randint(10000000000000) diff --git a/gym-unity/setup.py b/gym-unity/setup.py index 354b5e8b26f..fb6f81b8cb6 100755 --- a/gym-unity/setup.py +++ b/gym-unity/setup.py @@ -4,10 +4,10 @@ import sys from setuptools import setup, find_packages from setuptools.command.install import install -import gym_unity +from gym_unity import __version__, __release_tag__ -VERSION = gym_unity.__version__ -EXPECTED_TAG = gym_unity.__release_tag__ +VERSION = __version__ +EXPECTED_TAG = __release_tag__ class VerifyVersionCommand(install): @@ -38,6 +38,6 @@ def run(self): author_email="ML-Agents@unity3d.com", url="https://github.com/Unity-Technologies/ml-agents", packages=find_packages(), - install_requires=["gym", f"mlagents_envs=={VERSION}"], + install_requires=["gym>=0.11.0", f"mlagents_envs=={VERSION}", "certifi"], cmdclass={"verify": VerifyVersionCommand}, )