Added Python-Fire as Default Argument Parser

scripts/requirements.txt

@@ -8,3 +8,4 @@ pytest
 transformers
 datasets
 tensorboard
+fire 

src/components/TabTraining.vue

@@ -1,6 +1,13 @@
 <template>
  <div class="tab training">
  <h1>Training Options</h1>
+ <h2>Argument Parser</h2>
+ <FormSelect
+ :label="argparser.description"
+ :options="argparser.options"
+ :saveKey="argparser.name"
+ :defaultV="argparser.default"
+ />
  <h2 class="training">Deterministic Training</h2>
  <FormCheckbox
  :label="deterministic.description"
@@ -80,6 +87,7 @@ export default {
  saveConfig(deterministic.name, isDeterministic.value)
  })
  return {
+ ...training,
  store,
  deterministic,
  backend,

src/metadata/metadata.json

@@ -1,5 +1,12 @@
 {
  "training": {
+ "argparser": {
+ "name": "argparser",
+ "type": "array",
+ "description": "Select the argument parser for training",
+ "options": ["argparse", "python-fire"],
+ "default": "argparse"
+ },
  "deterministic": {
  "name": "deterministic",
  "type": "checkbox",

src/templates/template-common/README.md

@@ -143,7 +143,11 @@ nproc_per_node: #:::= it.nproc_per_node :::#
 ```
 
 ```sh
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+python main.py config.yaml --backend #:::= it.backend :::# [--override_arg=value]
+#::: } else { :::#
 python main.py config.yaml --backend #:::= it.backend :::#
+#::: } :::#
 ```
 
 #::: } :::#
@@ -155,7 +159,11 @@ python main.py config.yaml --backend #:::= it.backend :::#
 ### 1 GPU Training
 
 ```sh
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+python main.py config.yaml [--override_arg=value]
+#::: } else { :::#
 python main.py config.yaml
+#::: } :::#
 ```
 
 #::: } :::#

src/templates/template-common/main.py

@@ -1,19 +1,28 @@
 # main entrypoint
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+def main(config_path, output_path="./runs", render_mode=None, **kwargs):
+ config_path = Path(config_path)
+ assert config_path.exists(), config_path
+ config = setup_config(config_path, **kwargs)
+
+
+#::: } else { :::#
 def main():
  config = setup_config()
+ #::: } :::#
  #::: if (it.dist === 'spawn') { :::#
  #::: if (it.nproc_per_node && it.nnodes > 1 && it.master_addr && it.master_port) { :::#
- kwargs = {
+ kwargs_backend = {
  "nproc_per_node": config.nproc_per_node,
  "nnodes": config.nnodes,
  "node_rank": config.node_rank,
  "master_addr": config.master_addr,
  "master_port": config.master_port,
  }
  #::: } else if (it.nproc_per_node) { :::#
- kwargs = {"nproc_per_node": config.nproc_per_node}
+ kwargs_backend = {"nproc_per_node": config.nproc_per_node}
  #::: } :::#
- with idist.Parallel(config.backend, **kwargs) as p:
+ with idist.Parallel(config.backend, **kwargs_backend) as p:
  p.run(run, config=config)
  #::: } else { :::#
  with idist.Parallel(config.backend) as p:
@@ -22,4 +31,8 @@ def main():
 
 
 if __name__ == "__main__":
+ #::: if (['python-fire'].includes(it.argparser)) { :::#
+ fire.Fire(main)
+ #::: } else { :::#
  main()
+ #::: } :::#

src/templates/template-common/requirements.txt

@@ -2,6 +2,9 @@ torch>=1.10.2
 torchvision>=0.11.3
 pytorch-ignite>=0.4.8
 pyyaml
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+#:::= it.argparser :::#
+#::: } :::#
 
 #::: if (['neptune', 'polyaxon'].includes(it.logger)) { :::#
 

src/templates/template-common/utils.py

@@ -37,6 +37,37 @@
 from ignite.utils import setup_logger
 
 
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+
+
+class DotDict(dict):
+ """
+ Dictionary subclass that allows dot notation access to keys.
+ """
+
+ def __getattr__(self, attr):
+ value = self.get(attr)
+ if isinstance(value, dict):
+ return DotDict(value)
+ return value
+
+
+def setup_config(config_path, **kwargs):
+ with open(config_path, "r") as f:
+ config = yaml.safe_load(f.read())
+ for k, v in kwargs.items():
+ if k in config:
+ print(f"Override parameter {k}: {config[k]} -> {v}")
+ else:
+ print(f"{k} parameter not in {config_path}")
+ config[k] = v
+
+ return DotDict(config)
+
+
+#::: } else { :::#
+
+
 def get_default_parser():
  parser = ArgumentParser()
  parser.add_argument("config", type=Path, help="Config file path")
@@ -53,13 +84,8 @@ def get_default_parser():
 def setup_config(parser=None):
  if parser is None:
  parser = get_default_parser()
-
- args = parser.parse_args()
+ args = parser.parse_args()
  config_path = args.config
-
- with open(config_path, "r") as f:
- config = yaml.safe_load(f.read())
-
  optional_attributes = ["train_epoch_length", "eval_epoch_length"]
  for attr in optional_attributes:
  config[attr] = config.get(attr, None)
@@ -70,6 +96,9 @@ def setup_config(parser=None):
  return args
 
 
+#::: } :::#
+
+
 def log_metrics(engine: Engine, tag: str) -> None:
  """Log `engine.state.metrics` with given `engine` and `tag`.
 

src/templates/template-text-classification/main.py

@@ -1,6 +1,5 @@
 import os
 from pprint import pformat
-from shutil import copy
 from typing import Any
 
 import ignite.distributed as idist
@@ -13,6 +12,8 @@
 from torch import nn, optim
 from trainers import setup_evaluator, setup_trainer
 from utils import *
+import fire
+
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false" # remove tokenizer paralleism warning
 
@@ -25,7 +26,9 @@ def run(local_rank: int, config: Any):
  # create output folder and copy config file to output dir
  config.output_dir = setup_output_dir(config, rank)
  if rank == 0:
- copy(config.config, f"{config.output_dir}/config-lock.yaml")
+ with open(f"{config.output_dir}/config-lock.yaml", "a+") as f:
+ for key, value in config.items():
+ f.write(f"{key}: {value}\n")
 
  # donwload datasets and create dataloaders
  dataloader_train, dataloader_eval = setup_data(config)
@@ -69,7 +72,7 @@ def run(local_rank: int, config: Any):
  # setup engines logger with python logging
  # print training configurations
  logger = setup_logging(config)
- logger.info("Configuration: \n%s", pformat(vars(config)))
+ logger.info("Configuration: \n%s", pformat(config))
  trainer.logger = evaluator.logger = logger
 
  trainer.add_event_handler(Events.ITERATION_COMPLETED, lr_scheduler)

src/templates/template-text-classification/requirements.txt

@@ -4,6 +4,9 @@ pytorch-ignite>=0.4.8
 pyyaml
 transformers
 datasets
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+#:::= it.argparser :::#
+#::: } :::#
 
 #::: if (['neptune', 'polyaxon'].includes(it.logger)) { :::#
 

src/templates/template-vision-classification/main.py

@@ -1,5 +1,4 @@
 from pprint import pformat
-from shutil import copy
 from typing import Any
 
 import ignite.distributed as idist
@@ -12,6 +11,7 @@
 from torch import nn, optim
 from trainers import setup_evaluator, setup_trainer
 from utils import *
+import fire
 
 
 def run(local_rank: int, config: Any):
@@ -22,7 +22,9 @@ def run(local_rank: int, config: Any):
  # create output folder and copy config file to output dir
  config.output_dir = setup_output_dir(config, rank)
  if rank == 0:
- copy(config.config, f"{config.output_dir}/config-lock.yaml")
+ with open(f"{config.output_dir}/config-lock.yaml", "a+") as f:
+ for key, value in config.items():
+ f.write(f"{key}: {value}\n")
 
  # donwload datasets and create dataloaders
  dataloader_train, dataloader_eval = setup_data(config)
@@ -59,7 +61,7 @@ def run(local_rank: int, config: Any):
  # setup engines logger with python logging
  # print training configurations
  logger = setup_logging(config)
- logger.info("Configuration: \n%s", pformat(vars(config)))
+ logger.info("Configuration: \n%s", pformat(config))
  trainer.logger = evaluator.logger = logger
 
  trainer.add_event_handler(Events.ITERATION_COMPLETED, lr_scheduler)

src/templates/template-vision-classification/requirements.txt

@@ -2,6 +2,9 @@ torch>=1.10.2
 torchvision>=0.11.3
 pytorch-ignite>=0.4.8
 pyyaml
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+#:::= it.argparser :::#
+#::: } :::#
 
 #::: if (['neptune', 'polyaxon'].includes(it.logger)) { :::#
 

src/templates/template-vision-dcgan/main.py

@@ -1,5 +1,4 @@
 from pprint import pformat
-from shutil import copy
 from typing import Any
 
 import ignite.distributed as idist
@@ -13,6 +12,11 @@
 from trainers import setup_evaluator, setup_trainer
 from utils import *
 
+#::: if (it.
+# ) { :::#
+import fire
+
+
 FAKE_IMG_FNAME = "fake_sample_epoch_{:04d}.png"
 REAL_IMG_FNAME = "real_sample_epoch_{:04d}.png"
 
@@ -25,7 +29,9 @@ def run(local_rank: int, config: Any):
  # create output folder and copy config file to output dir
  config.output_dir = setup_output_dir(config, rank)
  if rank == 0:
- copy(config.config, f"{config.output_dir}/config-lock.yaml")
+ with open(f"{config.output_dir}/config-lock.yaml", "a+") as f:
+ for key, value in config.items():
+ f.write(f"{key}: {value}\n")
 
  # donwload datasets and create dataloaders
  dataloader_train, dataloader_eval, num_channels = setup_data(config)
@@ -74,7 +80,7 @@ def run(local_rank: int, config: Any):
  # setup engines logger with python logging
  # print training configurations
  logger = setup_logging(config)
- logger.info("Configuration: \n%s", pformat(vars(config)))
+ logger.info("Configuration: \n%s", pformat(config))
  trainer.logger = evaluator.logger = logger
 
  #::: if (it.save_training || it.save_evaluation) { :::#

src/templates/template-vision-dcgan/requirements.txt

@@ -2,6 +2,9 @@ torch>=1.10.2
 torchvision>=0.11.3
 pytorch-ignite>=0.4.8
 pyyaml
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+#:::= it.argparser :::#
+#::: } :::#
 
 #::: if (['neptune', 'polyaxon'].includes(it.logger)) { :::#
 

src/templates/template-vision-segmentation/main.py

@@ -1,6 +1,5 @@
 from functools import partial
 from pprint import pformat
-from shutil import copy
 from typing import Any, cast
 
 import ignite.distributed as idist
@@ -14,8 +13,10 @@
 from torch.optim.lr_scheduler import LambdaLR
 from trainers import setup_evaluator, setup_trainer
 from utils import *
+import fire
 from vis import predictions_gt_images_handler
 
+
 try:
  from torch.optim.lr_scheduler import LRScheduler as PyTorchLRScheduler
 except ImportError:
@@ -30,7 +31,9 @@ def run(local_rank: int, config: Any):
  # create output folder and copy config file to output dir
  config.output_dir = setup_output_dir(config, rank)
  if rank == 0:
- copy(config.config, f"{config.output_dir}/config-lock.yaml")
+ with open(f"{config.output_dir}/config-lock.yaml", "a+") as f:
+ for key, value in config.items():
+ f.write(f"{key}: {value}\n")
 
  # donwload datasets and create dataloaders
  dataloader_train, dataloader_eval = setup_data(config)
@@ -72,7 +75,7 @@ def run(local_rank: int, config: Any):
  # setup engines logger with python logging
  # print training configurations
  logger = setup_logging(config)
- logger.info("Configuration: \n%s", pformat(vars(config)))
+ logger.info("Configuration: \n%s", pformat(config))
  trainer.logger = evaluator.logger = logger
 
  if isinstance(lr_scheduler, PyTorchLRScheduler):

src/templates/template-vision-segmentation/requirements.txt

@@ -4,6 +4,11 @@ pytorch-ignite>=0.4.8
 pyyaml
 albumentations>=1.3.0
 image_dataset_viz
+#::: if (['python-fire'].includes(it.argparser)) { :::#
+
+#:::= it.argparser :::#
+
+#::: } :::#
 
 #::: if (['neptune', 'polyaxon'].includes(it.logger)) { :::#