Params for mbrl with dqn. (#1592)

Author: konradczechowski

tensor2tensor/models/research/rl.py

@@ -408,6 +408,34 @@ def dqn_original_params():
  return hparams
 
 
+@registry.register_hparams
+def dqn_guess1_params():
+ """Guess 1 for DQN params."""
+ hparams = dqn_atari_base()
+ hparams.set_hparam("num_frames", int(1e6))
+ hparams.set_hparam("agent_update_period", 1)
+ hparams.set_hparam("agent_target_update_period", 400)
+ # Small replay buffer size was set for mistake, but it seems to work
+ hparams.set_hparam("replay_buffer_replay_capacity", 10000)
+ return hparams
+
+
+@registry.register_hparams
+def dqn_2m_replay_buffer_params():
+ """Guess 1 for DQN params, 2 milions transitions in replay buffer"""
+ hparams = dqn_guess1_params()
+ hparams.set_hparam("replay_buffer_replay_capacity", int(2e6) + int(1e5))
+ return hparams
+
+
+@registry.register_hparams
+def dqn_10m_replay_buffer_params():
+ """Guess 1 for DQN params, 10 milions transitions in replay buffer"""
+ hparams = dqn_guess1_params()
+ hparams.set_hparam("replay_buffer_replay_capacity", int(10e6))
+ return hparams
+
+
 def rlmf_tiny_overrides():
  """Parameters to override for tiny setting excluding agent-related hparams."""
  return dict(

tensor2tensor/rl/trainer_model_based_params.py

@@ -210,6 +210,33 @@ def rlmb_dqn_base():
  return hparams
 
 
+@registry.register_hparams
+def rlmb_dqn_guess1():
+ """rlmb_dqn guess1 params"""
+ hparams = rlmb_dqn_base()
+ hparams.set_hparam("base_algo_params", "dqn_guess1_params")
+ # At the moment no other option for evaluation, so we want long rollouts to
+ # not bias scores.
+ hparams.set_hparam("eval_rl_env_max_episode_steps", 5000)
+ return hparams
+
+
+@registry.register_hparams
+def rlmb_dqn_guess1_2m_replay_buffer():
+ """rlmb_dqn guess1 params"""
+ hparams = rlmb_dqn_guess1()
+ hparams.set_hparam("base_algo_params", "dqn_2m_replay_buffer_params")
+ return hparams
+
+
+@registry.register_hparams
+def rlmb_dqn_guess1_10m_replay_buffer():
+ """rlmb_dqn guess1 params"""
+ hparams = rlmb_dqn_guess1()
+ hparams.set_hparam("base_algo_params", "dqn_10m_replay_buffer_params")
+ return hparams
+
+
 @registry.register_hparams
 def rlmb_basetest():
  """Base setting but quicker with only 2 epochs."""
@@ -617,6 +644,7 @@ def rlmb_dqn_tiny():
  hparams = rlmb_dqn_base()
  hparams = hparams.override_from_dict(_rlmb_tiny_overrides())
  update_hparams(hparams, dict(
+ base_algo_params="dqn_guess1_params",
  simulated_rollout_length=2,
  dqn_time_limit=2,
  dqn_num_frames=128,