Merge common and specific code-templates

.github/workflows/ci.yml

@@ -70,6 +70,7 @@ jobs:
  pip uninstall -y tqdm
  npm install -g pnpm
  pnpm i --frozen-lockfile --color
+ bash scripts/run_code_style.sh install
  # Show all installed dependencies
  pip list
 
@@ -81,6 +82,7 @@ jobs:
  - run: pnpm build
  - run: pnpm test:ci
  - run: sh ./scripts/run_tests.sh unzip
+ - run: pnpm lint
 
  - name: 'Run ${{ matrix.template }} ${{ matrix.test }}'
  run: sh ./scripts/run_tests.sh ${{ matrix.test }} ${{ matrix.template }}
@@ -108,5 +110,4 @@ jobs:
  - run: pip install -Uq pip wheel && bash scripts/run_code_style.sh install
  - run: npm install -g pnpm
  - run: pnpm i --frozen-lockfile --color
- - run: pnpm lint
- - run: python scripts/check_copies.py
+ - run: pnpm min_lint

.prettierignore

@@ -1,2 +1,3 @@
 dist
 pnpm-lock.yaml
+**/__DEV_CONFIG__.json

package.json

@@ -2,13 +2,14 @@
  "name": "@pytorch-ignite/code-generator",
  "version": "0.3.0",
  "scripts": {
- "dev": "vite",
+ "dev": "vite --port 5000",
  "build": "vite build",
  "serve": "vite preview",
- "test": "jest --color --runInBand",
+ "test": "rm -rf ./dist-tests && jest --color --runInBand",
  "test:ci": "start-server-and-test --expect 200 serve http://127.0.0.1:5000 test",
  "release": "node scripts/release.js",
  "fmt": "prettier --write . && bash scripts/run_code_style.sh fmt",
+ "min_lint": "prettier --check . && bash scripts/run_code_style.sh min_lint",
  "lint": "prettier --check . && bash scripts/run_code_style.sh lint"
  },
  "dependencies": {

scripts/run_code_style.sh

@@ -3,8 +3,12 @@
 set -xeu
 
 if [ $1 == "lint" ]; then
+ # Check that ./dist-tests/ exists and code is unzipped
+ ls ./dist-tests/vision-classification-all/main.py
+ ufmt diff .
+ flake8 --select F401,F821 ./dist-tests # find unused imports and non imported objects
+elif [ $1 == "min_lint" ]; then
  ufmt diff .
- flake8 --select F401 . # find unused imports
 elif [ $1 == "fmt" ]; then
  ufmt format .
 elif [ $1 == "install" ]; then

src/store.js

@@ -59,6 +59,15 @@ export function saveConfig(key, value) {
  }
 }
 
+// merges the code from the common and specific files using ejs
+function mergeCode(specificFileText, commonFileText) {
+ const replaced = specificFileText.replace(
+ /#::= from_template_common ::#/g,
+ commonFileText
+ )
+ return replaced
+}
+
 // render the code if there are fetched files for current selected template
 export function genCode() {
  const currentFiles = files[store.config.template]
@@ -78,6 +87,7 @@ export function genCode() {
  )
  // trim ` #`
  .replace(/\s{4}#$/gim, '')
+ .replace(/  # usort: skip/g, '')
  }
  if (isDev) {
  store.code[__DEV_CONFIG_FILE__] =
@@ -98,7 +108,14 @@ export async function fetchTemplates(template) {
  files[template] = {}
  for (const filename of templates[template]) {
  const response = await fetch(`${url}/${template}/${filename}`)
- files[template][filename] = await response.text()
+ const text_specific = await response.text()
+ // Dynamically fetch the common templates-code, if the file exists in common,
+ // then render the replace_here code tag using ejs template
+ // If the file doesn't exist in common, then it will fetch an empty string
+ // then the code tag is replaced with empty string
+ const res_common = await fetch(`${url}/template-common/${filename}`)
+ const text_common = await res_common.text()
+ files[template][filename] = mergeCode(text_specific, text_common)
  }
 
  // calling genCode explicitly here

src/templates/template-common/main.py

@@ -1,26 +1,3 @@
-ckpt_handler_train, ckpt_handler_eval = setup_handlers(
- trainer, evaluator, config, to_save_train, to_save_eval
-)
-
-#::: if (it.logger) { :::#
-if rank == 0:
- exp_logger.close()
-#::: } :::#
-
-#::: if (it.save_training || it.save_evaluation) { :::#
-# show last checkpoint names
-logger.info(
- "Last training checkpoint name - %s",
- ckpt_handler_train.last_checkpoint,
-)
-
-logger.info(
- "Last evaluation checkpoint name - %s",
- ckpt_handler_eval.last_checkpoint,
-)
-#::: } :::#
-
-
 # main entrypoint
 def main():
  config = setup_config()
@@ -42,3 +19,7 @@ def main():
  with idist.Parallel(config.backend) as p:
  p.run(run, config=config)
  #::: } :::#
+
+
+if __name__ == "__main__":
+ main()

src/templates/template-common/utils.py

@@ -10,11 +10,34 @@
 import yaml
 from ignite.contrib.engines import common
 from ignite.engine import Engine
+
+#::: if (it.save_training || it.save_evaluation || it.patience || it.terminate_on_nan || it.limit_sec) { :::#
 from ignite.engine.events import Events
-from ignite.handlers import Checkpoint, DiskSaver, global_step_from_engine
+
+#::: } :::#
+#::: if (it.save_training || it.save_evaluation) { :::#
+from ignite.handlers import (
+ Checkpoint,
+ DiskSaver,
+ global_step_from_engine,
+) # usort: skip
+
+#::: } else { :::#
+from ignite.handlers import Checkpoint
+
+#::: } :::#
+#::: if (it.patience) { :::#
 from ignite.handlers.early_stopping import EarlyStopping
+
+#::: } :::#
+#::: if (it.terminate_on_nan) { :::#
 from ignite.handlers.terminate_on_nan import TerminateOnNan
+
+#::: } :::#
+#::: if (it.limit_sec) { :::#
 from ignite.handlers.time_limit import TimeLimit
+
+#::: } :::#
 from ignite.utils import setup_logger
 
 
@@ -141,72 +164,6 @@ def setup_logging(config: Any) -> Logger:
  return logger
 
 
-#::: if (it.save_training || it.save_evaluation || it.patience || it.terminate_on_nan || it.limit_sec) { :::#
-
-
-def setup_handlers(
- trainer: Engine,
- evaluator: Engine,
- config: Any,
- to_save_train: Optional[dict] = None,
- to_save_eval: Optional[dict] = None,
-):
- """Setup Ignite handlers."""
-
- ckpt_handler_train = ckpt_handler_eval = None
- #::: if (it.save_training || it.save_evaluation) { :::#
- # checkpointing
- saver = DiskSaver(config.output_dir / "checkpoints", require_empty=False)
- #::: if (it.save_training) { :::#
- ckpt_handler_train = Checkpoint(
- to_save_train,
- saver,
- filename_prefix=config.filename_prefix,
- n_saved=config.n_saved,
- )
- trainer.add_event_handler(
- Events.ITERATION_COMPLETED(every=config.save_every_iters),
- ckpt_handler_train,
- )
- #::: } :::#
- #::: if (it.save_evaluation) { :::#
- global_step_transform = None
- if to_save_train.get("trainer", None) is not None:
- global_step_transform = global_step_from_engine(to_save_train["trainer"])
- ckpt_handler_eval = Checkpoint(
- to_save_eval,
- saver,
- filename_prefix="best",
- n_saved=config.n_saved,
- global_step_transform=global_step_transform,
- )
- evaluator.add_event_handler(Events.EPOCH_COMPLETED(every=1), ckpt_handler_eval)
- #::: } :::#
- #::: } :::#
-
- #::: if (it.patience) { :::#
- # early stopping
-
- es = EarlyStopping(config.patience, score_fn, trainer)
- evaluator.add_event_handler(Events.EPOCH_COMPLETED, es)
- #::: } :::#
-
- #::: if (it.terminate_on_nan) { :::#
- # terminate on nan
- trainer.add_event_handler(Events.ITERATION_COMPLETED, TerminateOnNan())
- #::: } :::#
-
- #::: if (it.limit_sec) { :::#
- # time limit
- trainer.add_event_handler(Events.ITERATION_COMPLETED, TimeLimit(config.limit_sec))
- #::: } :::#
- #::: if (it.save_training || it.save_evaluation) { :::#
- return ckpt_handler_train, ckpt_handler_eval
- #::: } :::#
-
-
-#::: } :::#
-
 #::: if (it.logger) { :::#
 
 

src/templates/template-text-classification/config.yaml

@@ -1,13 +1,4 @@
-seed: 666
-data_path: ./
-train_batch_size: 32
-eval_batch_size: 32
-num_workers: 4
-max_epochs: 20
-train_epoch_length: 1000
-eval_epoch_length: 1000
-use_amp: false
-debug: false
+#::= from_template_common ::#
 model: bert-base-uncased
 model_dir: /tmp/model
 tokenizer_dir: /tmp/tokenizer
@@ -18,45 +9,3 @@ weight_decay: 0.01
 num_warmup_epochs: 0
 max_length: 256
 lr: 0.00005
-
-#::: if (it.dist === 'spawn') { :::#
-# distributed spawn
-nproc_per_node: #:::= it.nproc_per_node :::#
-#::: if (it.nnodes) { :::#
-# distributed multi node spawn
-nnodes: #:::= it.nnodes :::#
-#::: if (it.nnodes > 1) { :::#
-node_rank: 0
-master_addr: #:::= it.master_addr :::#
-master_port: #:::= it.master_port :::#
-#::: } :::#
-#::: } :::#
-#::: } :::#
-
-#::: if (it.filename_prefix) { :::#
-filename_prefix: #:::= it.filename_prefix :::#
-#::: } :::#
-
-#::: if (it.n_saved) { :::#
-n_saved: #:::= it.n_saved :::#
-#::: } :::#
-
-#::: if (it.save_every_iters) { :::#
-save_every_iters: #:::= it.save_every_iters :::#
-#::: } :::#
-
-#::: if (it.patience) { :::#
-patience: #:::= it.patience :::#
-#::: } :::#
-
-#::: if (it.limit_sec) { :::#
-limit_sec: #:::= it.limit_sec :::#
-#::: } :::#
-
-#::: if (it.output_dir) { :::#
-output_dir: #:::= it.output_dir :::#
-#::: } :::#
-
-#::: if (it.log_every_iters) { :::#
-log_every_iters: #:::= it.log_every_iters :::#
-#::: } :::#

src/templates/template-text-classification/main.py

@@ -173,28 +173,4 @@ def _():
  #::: } :::#
 
 
-# main entrypoint
-def main():
- config = setup_config()
- #::: if (it.dist === 'spawn') { :::#
- #::: if (it.nproc_per_node && it.nnodes > 1 && it.master_addr && it.master_port) { :::#
- kwargs = {
- "nproc_per_node": config.nproc_per_node,
- "nnodes": config.nnodes,
- "node_rank": config.node_rank,
- "master_addr": config.master_addr,
- "master_port": config.master_port,
- }
- #::: } else if (it.nproc_per_node) { :::#
- kwargs = {"nproc_per_node": config.nproc_per_node}
- #::: } :::#
- with idist.Parallel(config.backend, **kwargs) as p:
- p.run(run, config=config)
- #::: } else { :::#
- with idist.Parallel(config.backend) as p:
- p.run(run, config=config)
- #::: } :::#
-
-
-if __name__ == "__main__":
- main()
+#::= from_template_common ::#