Develop magic string + trajectory (#3122)

* added team id and identifier concat to behavior parameters * splitting brain params into brain name and identifiers * set team id in prefab * recieves brain_name and identifier on python side * added team id and identifier concat to behavior parameters * splitting brain params into brain name and identifiers * set team id in prefab * recieves brain_name and identifier on python side * rebased with develop * Correctly calls concatBehaviorIdentifiers * added team id and identifier concat to behavior parameters * splitting brain params into brain name and identifiers * set team id in prefab * recieves brain_name and identifier on python side * rebased with develop * Correctly calls concatBehaviorIdentifiers * trainer_controller expects name_behavior_ids * add_policy and create_policy separated * adjusting tests to expect trainer.add_policy to be called * fixing tests * fixed naming name_behavior_id * added team id and identifier concat to behavior parameters * splitting brain params into brain name and identifiers * set team id in prefab * added team id and identifier concat to behavior parameters * recieves brain_name and identifier on python side * splitting brain params into brain name and identifiers * recieves brain_name and identifier on python side * rebased with develop * Correctly calls concatBehaviorIdentifiers * splitting brain params into brain name and identifiers * recieves brain_name and identifier on python side * rebased with develop * Correctly calls concatBehaviorIdentifiers * trainer_controller expects name_behavior_ids * add_policy and create_policy separated * adjusting tests to expect trainer.add_policy to be called * fixing tests * fixed naming name_behavior_id * passes all pytest and C# tests * fixed printing nonsense * fixed assets that got messed up * added ppo/sac_policy attributes to keep up with master * fixing ci tests * fixing more ci tests * fixed default trainer_util test to expect brain_name * fixing ci ppo_policy * fixed more ci problems/removed self.policies * Add agent group name to Trajectory * Rename to behavior_id * magic string protocol with trainer refactor * fixed logger warning * removed self.policy from rl_trainer * removed self.trainer from trainer.py * fixed increment_step tests * fixing circleci tests * type annotations to the trainer params * parameters descriptions for trainers Co-authored-by: Ervin T. <ervin@unity3d.com>

Author: andrewcoh

UnitySDK/Assets/ML-Agents/Editor/BehaviorParametersEditor.cs

@@ -33,6 +33,7 @@ public override void OnInspectorGUI()
  EditorGUILayout.PropertyField(so.FindProperty("m_InferenceDevice"), true);
  EditorGUI.indentLevel--;
  EditorGUILayout.PropertyField(so.FindProperty("m_BehaviorType"));
+ EditorGUILayout.PropertyField(so.FindProperty("m_TeamID"));
  EditorGUILayout.PropertyField(so.FindProperty("m_useChildSensors"), true);
  // EditorGUILayout.PropertyField(serializedObject.FindProperty("m_Heuristic"), true);
  EditorGUI.indentLevel--;

UnitySDK/Assets/ML-Agents/Scripts/Policy/BehaviorParameters.cs

@@ -1,5 +1,6 @@
 using Barracuda;
 using System;
+using System.Collections.Generic;
 using UnityEngine;
 
 namespace MLAgents
@@ -34,6 +35,8 @@ private enum BehaviorType
  [HideInInspector]
  [SerializeField]
  string m_BehaviorName = "My Behavior";
+ [HideInInspector] [SerializeField]
+ int m_TeamID = 0;
  [HideInInspector]
  [SerializeField]
  [Tooltip("Use all Sensor components attached to child GameObjects of this Agent.")]
@@ -51,7 +54,9 @@ public bool useChildSensors
 
  public string behaviorName
  {
- get { return m_BehaviorName; }
+ 
+ get { return m_BehaviorName + "?team=" + m_TeamID;} 
+
  }
 
  public IPolicy GeneratePolicy(Func<float[]> heuristic)
@@ -65,7 +70,7 @@ public IPolicy GeneratePolicy(Func<float[]> heuristic)
  case BehaviorType.Default:
  if (FindObjectOfType<Academy>().IsCommunicatorOn)
  {
- return new RemotePolicy(m_BrainParameters, m_BehaviorName);
+ return new RemotePolicy(m_BrainParameters, behaviorName);
  }
  if (m_Model != null)
  {

ml-agents/mlagents/trainers/agent_processor.py

@@ -21,6 +21,7 @@ def __init__(
  self,
  trainer: Trainer,
  policy: TFPolicy,
+ behavior_id: str,
  stats_reporter: StatsReporter,
  max_trajectory_length: int = sys.maxsize,
  ):
@@ -44,6 +45,7 @@ def __init__(
  self.stats_reporter = stats_reporter
  self.trainer = trainer
  self.max_trajectory_length = max_trajectory_length
+ self.behavior_id = behavior_id
 
  def add_experiences(
  self,
@@ -133,6 +135,7 @@ def add_experiences(
  steps=self.experience_buffers[agent_id],
  agent_id=agent_id,
  next_obs=next_obs,
+ behavior_id=self.behavior_id,
  )
  # This will eventually be replaced with a queue
  self.trainer.process_trajectory(trajectory)

ml-agents/mlagents/trainers/ppo/trainer.py

@@ -10,6 +10,8 @@
 from mlagents.trainers.ppo.policy import PPOPolicy
 from mlagents.trainers.ppo.multi_gpu_policy import MultiGpuPPOPolicy, get_devices
 from mlagents.trainers.rl_trainer import RLTrainer
+from mlagents.trainers.brain import BrainParameters
+from mlagents.trainers.tf_policy import TFPolicy
 from mlagents.trainers.trajectory import Trajectory
 
 logger = logging.getLogger("mlagents.trainers")
@@ -20,26 +22,28 @@ class PPOTrainer(RLTrainer):
 
  def __init__(
  self,
- brain,
- reward_buff_cap,
- trainer_parameters,
- training,
- load,
- seed,
- run_id,
- multi_gpu,
+ brain_name: str,
+ reward_buff_cap: int,
+ trainer_parameters: dict,
+ training: bool,
+ load: bool,
+ seed: int,
+ run_id: str,
+ multi_gpu: bool,
  ):
  """
  Responsible for collecting experiences and training PPO model.
- :param trainer_parameters: The parameters for the trainer (dictionary).
+ :param brain_name: The name of the brain associated with trainer config
  :param reward_buff_cap: Max reward history to track in the reward buffer
+ :param trainer_parameters: The parameters for the trainer (dictionary).
  :param training: Whether the trainer is set for training.
  :param load: Whether the model should be loaded.
  :param seed: The seed the model will be initialized with
  :param run_id: The identifier of the current run
+ :param multi_gpu: Boolean for multi-gpu policy model
  """
  super(PPOTrainer, self).__init__(
- brain, trainer_parameters, training, run_id, reward_buff_cap
+ brain_name, trainer_parameters, training, run_id, reward_buff_cap
  )
  self.param_keys = [
  "batch_size",
@@ -63,19 +67,10 @@ def __init__(
  "reward_signals",
  ]
  self.check_param_keys()
-
- if multi_gpu and len(get_devices()) > 1:
- self.ppo_policy = MultiGpuPPOPolicy(
- seed, brain, trainer_parameters, self.is_training, load
- )
- else:
- self.ppo_policy = PPOPolicy(
- seed, brain, trainer_parameters, self.is_training, load
- )
- self.policy = self.ppo_policy
-
- for _reward_signal in self.policy.reward_signals.keys():
- self.collected_rewards[_reward_signal] = defaultdict(lambda: 0)
+ self.load = load
+ self.multi_gpu = multi_gpu
+ self.seed = seed
+ self.policy: TFPolicy = None
 
  def process_trajectory(self, trajectory: Trajectory) -> None:
  """
@@ -161,7 +156,9 @@ def process_trajectory(self, trajectory: Trajectory) -> None:
 
  # If this was a terminal trajectory, append stats and reset reward collection
  if trajectory.done_reached:
- self._update_end_episode_stats(agent_id)
+ self._update_end_episode_stats(
+ agent_id, self.get_policy(trajectory.behavior_id)
+ )
 
  def is_ready_update(self):
  """
@@ -218,6 +215,56 @@ def update_policy(self):
  self.stats_reporter.add_stat(stat, val)
  self.clear_update_buffer()
 
+ def create_policy(self, brain_parameters: BrainParameters) -> TFPolicy:
+ """
+ Creates a PPO policy to trainers list of policies.
+ :param brain_parameters: specifications for policy construction
+ :return policy
+ """
+
+ if self.multi_gpu and len(get_devices()) > 1:
+ policy: PPOPolicy = MultiGpuPPOPolicy(
+ self.seed,
+ brain_parameters,
+ self.trainer_parameters,
+ self.is_training,
+ self.load,
+ )
+ else:
+ policy = PPOPolicy(
+ self.seed,
+ brain_parameters,
+ self.trainer_parameters,
+ self.is_training,
+ self.load,
+ )
+
+ for _reward_signal in policy.reward_signals.keys():
+ self.collected_rewards[_reward_signal] = defaultdict(lambda: 0)
+
+ return policy
+
+ def add_policy(self, name_behavior_id: str, policy: TFPolicy) -> None:
+ """
+ Adds policy to trainer.
+ :param brain_parameters: specifications for policy construction
+ """
+ if self.policy:
+ logger.warning(
+ "add_policy has been called twice. {} is not a multi-agent trainer".format(
+ self.__class__.__name__
+ )
+ )
+ self.policy = policy
+
+ def get_policy(self, name_behavior_id: str) -> TFPolicy:
+ """
+ Gets policy from trainer associated with name_behavior_id
+ :param name_behavior_id: full identifier of policy
+ """
+
+ return self.policy
+
 
 def discount_rewards(r, gamma=0.99, value_next=0.0):
  """

ml-agents/mlagents/trainers/rl_trainer.py

@@ -3,6 +3,7 @@
 from typing import Dict
 from collections import defaultdict
 
+from mlagents.trainers.tf_policy import TFPolicy
 from mlagents.trainers.buffer import AgentBuffer
 from mlagents.trainers.trainer import Trainer, UnityTrainerException
 from mlagents.trainers.components.reward_signals import RewardSignalResult
@@ -47,7 +48,7 @@ def end_episode(self) -> None:
  for agent_id in rewards:
  rewards[agent_id] = 0
 
- def _update_end_episode_stats(self, agent_id: str) -> None:
+ def _update_end_episode_stats(self, agent_id: str, policy: TFPolicy) -> None:
  self.episode_steps[agent_id] = 0
  for name, rewards in self.collected_rewards.items():
  if name == "environment":
@@ -58,7 +59,7 @@ def _update_end_episode_stats(self, agent_id: str) -> None:
  rewards[agent_id] = 0
  else:
  self.stats_reporter.add_stat(
- self.policy.reward_signals[name].stat_name, rewards.get(agent_id, 0)
+ policy.reward_signals[name].stat_name, rewards.get(agent_id, 0)
  )
  rewards[agent_id] = 0