feat: archive as python package system

app/codegen.py

@@ -12,7 +12,7 @@ def __init__(self, templates_dir: str = "./templates", dist_dir: str = "./dist")
  self.dist_dir = Path(dist_dir)
  self.template_list = [p.stem for p in self.templates_dir.iterdir() if p.is_dir() and not p.stem.startswith("_")]
  self.rendered_code = {t: {} for t in self.template_list}
- self.available_archive_formats = sorted(map(lambda x: x[0], shutil.get_archive_formats()), reverse=True)
+ self.available_archive_formats = [x[0] for x in shutil.get_archive_formats()[::-1]]
 
  def render_templates(self, template_name: str, config: dict):
  """Renders all the templates files from template folder for the given config."""
@@ -31,27 +31,31 @@ def render_templates(self, template_name: str, config: dict):
  self.rendered_code[template_name][fname] = code
  yield fname, code
 
- def mk_dist_template_dir(self, template_name: str):
- self.dist_template_dir = Path(f"{self.dist_dir}/{template_name}")
- self.dist_template_dir.mkdir(parents=True, exist_ok=True)
+ def make_and_write(self, template_name: str):
+ """Make the directories first and write to the files"""
+ for p in (self.templates_dir / template_name).rglob("*"):
+ if not p.stem.startswith("_") and p.is_dir():
+ # p is templates/template_name/...
+ # remove "templates" from p.parts and join with "/", so we'll have
+ # template_name/...
+ p = "/".join(p.parts[1:])
+ else:
+ p = template_name
 
- def write_file(self, fname: str, code: str) -> None:
- """Creates `fname` with content `code` in `dist_dir/template_name`."""
- (self.dist_template_dir / fname).write_text(code)
+ if not (self.dist_dir / p).is_dir():
+ (self.dist_dir / p).mkdir(parents=True, exist_ok=True)
 
- def write_files(self, template_name):
- """Writes all rendered code for the specified template."""
- # Save files with rendered code to the disk
  for fname, code in self.rendered_code[template_name].items():
- self.write_file(fname, code)
+ (self.dist_dir / template_name / fname).write_text(code)
 
  def make_archive(self, template_name, archive_format):
  """Creates dist dir with generated code, then makes the archive."""
- self.mk_dist_template_dir(template_name)
- self.write_files(template_name)
+
+ self.make_and_write(template_name)
  archive_fname = shutil.make_archive(
- base_name=str(self.dist_template_dir),
+ base_name=template_name,
+ root_dir=self.dist_dir,
  format=archive_format,
- base_dir=self.dist_template_dir,
+ base_dir=template_name,
  )
- return archive_fname
+ return shutil.move(archive_fname, self.dist_dir / archive_fname.split("/")[-1])

app/streamlit_app.py

@@ -1,4 +1,3 @@
-import os
 import shutil
 from pathlib import Path
 from subprocess import check_output
@@ -10,6 +9,13 @@
 __version__ = "0.1.0"
 
 
+FOLDER_TO_TEMPLATE_NAME = {
+ "Single Model, Single Optimizer": "single",
+ "Generative Adversarial Network": "gan",
+ "Image Classification": "image_classification",
+}
+
+
 class App:
  page_title = "Code Generator"
  page_icon = "https://raw.githubusercontent.com/pytorch/ignite/master/assets/logo/ignite_logomark.svg"
@@ -39,18 +45,19 @@ def sidebar(self, template_list=None, config=None):
  template_list = template_list or []
  st.markdown("### Choose a Template")
  self.template_name = st.selectbox("Available Templates are:", options=template_list)
+ self.template_name = FOLDER_TO_TEMPLATE_NAME[self.template_name]
  with st.sidebar:
  if self.template_name:
  config = config(self.template_name)
  self.config = config.get_configs()
  else:
  self.config = {}
 
- def render_code(self, fname="", code=""):
+ def render_code(self, fname: str = "", code: str = ""):
  """Main content with the code."""
- with st.beta_expander(f"View rendered {fname}"):
+ with st.beta_expander(f"View rendered {fname}", expanded=fname.endswith(".md")):
  if fname.endswith(".md"):
- st.markdown(code)
+ st.markdown(code, unsafe_allow_html=True)
  else:
  st.code(code)
 
@@ -63,12 +70,12 @@ def add_sidebar(self):
  def config(template_name):
  return import_from_file("template_config", f"./templates/{template_name}/_sidebar.py")
 
- self.sidebar(self.codegen.template_list, config)
+ self.sidebar([*FOLDER_TO_TEMPLATE_NAME], config)
 
  def add_content(self):
  """Get generated/rendered code from the codegen."""
  content = [*self.codegen.render_templates(self.template_name, self.config)]
- if st.checkbox("View rendered code ?"):
+ if st.checkbox("View rendered code ?", value=True):
  for fname, code in content:
  self.render_code(fname, code)
 
@@ -90,7 +97,7 @@ def add_download(self):
  shutil.copy(archive_fname, dist_path)
  st.success(f"Download link : [{archive_fname}](./static/{archive_fname})")
  with col2:
- self.render_directory(os.path.join(self.codegen.dist_dir, self.template_name))
+ self.render_directory(Path(self.codegen.dist_dir, self.template_name))
 
  def run(self):
  self.add_sidebar()

templates/single/utils.pyi → templates/_base/_argparse.pyi

@@ -1,65 +1,14 @@
+{% block imports %}
 from argparse import ArgumentParser
+{% endblock %}
 
+{% block defaults %}
 DEFAULTS = {
- # dataset options
- "dataset": {
- "default": "{{ dataset }}",
- "type": str,
- "choices": ["cifar10", "lsun", "imagenet", "folder", "lfw", "fake", "mnist"],
- "help": "dataset to use ({{ dataset }})",
- },
- "data_path": {
- "default": "{{ data_path }}",
- "type": str,
- "help": "datasets path ({{ data_path }})",
- },
- # dataloader options
- "batch_size": {
- "default": {{batch_size}},
- "type": int,
- "help": "will be equally divided by number of GPUs if in distributed ({{ batch_size }})",
- },
- "num_workers": {
- "default": {{num_workers}},
- "type": int,
- "help": "num_workers for DataLoader ({{ num_workers }})",
- },
- # optimizer options
- "beta_1": {
- "default": {{beta_1}},
- "type": float,
- "help": "beta_1 for Adam optimizer ({{ beta_1 }})",
- },
- "lr": {
- "default": {{lr}},
- "type": float,
- "help": "learning rate used by torch.optim.* ({{ lr }})",
- },
- # training options
- "max_epochs": {
- "default": {{max_epochs}},
- "type": int,
- "help": "max_epochs of ignite.Engine.run() for training ({{ max_epochs }})",
- },
- "log_train": {
- "default": {{log_train}},
- "type": int,
- "help": "logging interval of training iteration ({{ log_train }})",
- },
- "seed": {
- "default": {{seed}},
- "type": int,
- "help": "used in ignite.utils.manual_seed() ({{ seed }})",
- },
- "epoch_length": {
- "default": None,
- "type": int,
- "help": "epoch_length of ignite.Engine.run() for training (None)",
- },
  "verbose": {
  "action": "store_true",
  "help": "use logging.INFO in ignite.utils.setup_logger",
  },
+
  # distributed training options
  "nproc_per_node": {
  "default": {{nproc_per_node}},
@@ -88,16 +37,17 @@ DEFAULTS = {
  "type": int,
  "help": "master node port for torch native backends {{ master_port }}",
  },
+
  # ignite handlers options
  "output_path": {
  "default": "{{output_path}}",
  "type": str,
- "help": "output path to indicate where to_save objects are stored",
+ "help": "output path to indicate where to_save objects are stored ({{output_path}})",
  },
  "save_every_iters": {
  "default": {{save_every_iters}},
  "type": int,
- "help": "Saving iteration interval",
+ "help": "Saving iteration interval ({{save_every_iters}})",
  },
  "n_saved": {
  "default": {{n_saved}},
@@ -107,73 +57,60 @@ DEFAULTS = {
  "log_every_iters": {
  "default": {{log_every_iters}},
  "type": int,
- "help": "logging interval for iteration progress bar",
+ "help": "logging interval for iteration progress bar ({{log_every_iters}})",
  },
  "with_pbars": {
  "default": {{with_pbars}},
  "type": bool,
- "help": "show epoch-wise and iteration-wise progress bars",
+ "help": "show epoch-wise and iteration-wise progress bars ({{with_pbars}})",
  },
  "with_pbar_on_iters": {
  "default": {{with_pbar_on_iters}},
  "type": bool,
- "help": "show iteration progress bar or not",
+ "help": "show iteration progress bar or not ({{with_pbar_on_iters}})",
  },
  "stop_on_nan": {
  "default": {{stop_on_nan}},
  "type": bool,
- "help": "stop the training if engine output contains NaN/inf values",
+ "help": "stop the training if engine output contains NaN/inf values (stop_on_nan)",
  },
  "clear_cuda_cache": {
  "default": {{clear_cuda_cache}},
  "type": bool,
- "help": "clear cuda cache every end of epoch",
+ "help": "clear cuda cache every end of epoch ({{clear_cuda_cache}})",
  },
- # ignite logger options
- "logger_log_every_iters": {
- "default": {{logger_log_every_iters}},
+ "with_gpu_stats": {
+ "default": {{with_gpu_stats}},
+ "type": bool,
+ "help": "show gpu information, requires pynvml ({{with_gpu_stats}})",
+ },
+ "patience": {
+ "default": {{patience}},
  "type": int,
- "help": "logging interval for experiment tracking system ({{logger_log_every_iters}})",
+ "help": "number of events to wait if no improvement and then stop the training ({{patience}})"
+ },
+ "limit_sec": {
+ "default": {{limit_sec}},
+ "type": int,
+ "help": "maximum time before training terminates in seconds ({{limit_sec}})"
  },
+
+ # ignite logger options
  "filepath": {
  "default": "{{ filepath }}",
  "type": str,
  "help": "logging file path ({{ filepath }})",
  },
- # model options
- "z_dim": {
- "default": {{z_dim}},
- "type": int,
- "help": "size of the latent z vector ({{ z_dim }})",
- },
- "alpha": {
- "default": {{alpha}},
- "type": float,
- "help": "running average decay factor ({{ alpha }})",
- },
- "g_filters": {
- "default": {{g_filters}},
- "type": int,
- "help": "number of filters in the second-to-last generator deconv layer ({{ g_filters }})",
- },
- "d_filters": {
- "default": {{d_filters}},
+ "logger_log_every_iters": {
+ "default": {{logger_log_every_iters}},
  "type": int,
- "help": "number of filters in first discriminator conv layer ({{ d_filters }})",
- },
- "saved_G": {
- "default": {{saved_G}},
- "type": str,
- "help": "path to saved generator ({{ saved_G }})",
- },
- "saved_D": {
- "default": {{saved_D}},
- "type": str,
- "help": "path to saved discriminator ({{ saved_D }})",
+ "help": "logging interval for experiment tracking system ({{logger_log_every_iters}})",
  },
 }
+{% endblock %}
 
 
+{% block get_default_parser %}
 def get_default_parser():
  """Get the default configs for training."""
  parser = ArgumentParser(add_help=False)
@@ -182,3 +119,4 @@ def get_default_parser():
  parser.add_argument(f"--{key}", **value)
 
  return parser
+{% endblock %}

templates/_base/_handlers.pyi

@@ -15,6 +15,8 @@ def get_handlers(
  model: Module,
  train_engine: Engine,
  eval_engine: Engine,
+ metric_name: str,
+ es_metric_name: str,
  train_sampler: Optional[DistributedSampler] = None,
  to_save: Optional[Mapping] = None,
  lr_scheduler: Optional[LRScheduler] = None,
@@ -35,18 +37,18 @@ def get_handlers(
  - `stop_on_nan`: Stop the training if engine output contains NaN/inf values
  - `clear_cuda_cache`: clear cuda cache every end of epoch
  - `with_gpu_stats`: show GPU information: used memory percentage, gpu utilization percentage values
- - `metric_name`: evaluation metric to save the best model
- - `patience`: number of events to wait if no improvement and then stop the training.
- - `es_metric_name`: evaluation metric to early stop the model
- - `limit_sec`: 
+ - `patience`: number of events to wait if no improvement and then stop the training
+ - `limit_sec`: maximum time before training terminates in seconds
 
  model: best model to save
  train_engine: the engine used for training
  eval_engine: the engine used for evaluation
+ metric_name: evaluation metric to save the best model
+ es_metric_name: evaluation metric to early stop the model
  train_sampler: distributed training sampler to call `set_epoch`
  to_save: objects to save during training
  lr_scheduler: learning rate scheduler as native torch LRScheduler or ignite’s parameter scheduler
- output_names: list of names associated with `train_engine`'s process_function output dictionary.
+ output_names: list of names associated with `train_engine`'s process_function output dictionary
 
  Returns:
  best_model_handler, es_handler, timer_handler
@@ -78,7 +80,7 @@ def get_handlers(
  output_path=config.output_path,
  evaluator=eval_engine,
  model=model,
- metric_name=config.metric_name,
+ metric_name=metric_name,
  n_saved=config.n_saved,
  trainer=train_engine,
  tag='eval',
@@ -91,7 +93,7 @@ def get_handlers(
  patience=config.patience,
  evaluator=eval_engine,
  trainer=train_engine,
- metric_name=config.es_metric_name,
+ metric_name=es_metric_name,
  )
  {% endif %}
  {% if setup_timer %}