Merge branch 'main' into arrayschema_typedsl_needed

Author: mr-c

.codecov.yml

@@ -0,0 +1,3 @@
+codecov:
+ notify:
+ wait_for_ci: yes

.coveragerc

@@ -1,6 +1,6 @@
 [run]
 branch = True
-source = cwltool
+source_pkgs = cwltool
 omit = cwltool/run_job.py
 
 [report]

.dockerignore

@@ -18,6 +18,7 @@ testenv*/
 .github/
 cwltool/*.so
 *.Dockerfile
+tmp/
 build-cwltool-docker.sh
 __pycache__/
 */__pycache__/
@@ -26,3 +27,4 @@ __pycache__/
 *.orig
 .dockerignore
 cache*
+*.swp

.github/workflows/ci-tests.yml

@@ -7,8 +7,12 @@ on:
  branches: [ main ]
  workflow_dispatch:
 
+concurrency:
+ group: build-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
 env:
- singularity_version: 3.6.4
+ singularity_version: 3.8.3
 
 jobs:
 
@@ -20,7 +24,7 @@ jobs:
  strategy:
  matrix:
  py-ver-major: [3]
- py-ver-minor: [6, 7, 8, 9]
+ py-ver-minor: [7, 8, 9, 10]
  step: [lint, unit, bandit, mypy]
 
  env:
@@ -34,7 +38,7 @@ jobs:
 
  - name: Set up Singularity
  if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
- uses: eWaterCycle/setup-singularity@v6
+ uses: eWaterCycle/setup-singularity@v7
  with:
  singularity-version: ${{ env.singularity_version }}
 
@@ -46,12 +50,10 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: ${{ env.py-semver }}
-
- - name: Cache for pip
- uses: actions/cache@v2
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt', 'tox.ini') }}
+ cache: pip
+ cache-dependency-path: |
+ requirements.txt
+ tox.ini
 
  - name: Upgrade setuptools and install tox
  run: |
@@ -70,7 +72,7 @@ jobs:
 
  - name: Upload coverage to Codecov
  if: ${{ matrix.step == 'unit' }}
- uses: codecov/codecov-action@v2.0.3
+ uses: codecov/codecov-action@v2.1.0
  with:
  fail_ci_if_error: true
 
@@ -84,8 +86,8 @@ jobs:
  step: [lintreadme, shellcheck, pydocstyle]
 
  env:
- py-semver: 3.9
- TOXENV: ${{ format('py39-{0}', matrix.step) }}
+ py-semver: "3.10"
+ TOXENV: ${{ format('py310-{0}', matrix.step) }}
 
  steps:
  - uses: actions/checkout@v2
@@ -96,12 +98,7 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: ${{ env.py-semver }}
-
- - name: Cache for pip
- uses: actions/cache@v2
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt') }}
+ cache: pip
 
  - name: Upgrade setuptools and install tox
  run: |
@@ -123,26 +120,21 @@ jobs:
  strategy:
  matrix:
  cwl-version: [v1.0, v1.1, v1.2]
- container: [docker, singularity]
+ container: [docker, singularity, podman]
 
  steps:
  - uses: actions/checkout@v2
 
  - name: Set up Singularity
- uses: eWaterCycle/setup-singularity@v6
+ uses: eWaterCycle/setup-singularity@v7
  with:
  singularity-version: ${{ env.singularity_version }}
 
  - name: Set up Python
  uses: actions/setup-python@v2
  with:
  python-version: 3.9
-
- - name: Cache for pip
- uses: actions/cache@v2
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-conformance-${{ matrix.step }}-${{ hashFiles('requirements.txt') }}
+ cache: pip
 
  - name: Run CWL conformance tests ${{ matrix.cwl-version }}
  env:
@@ -160,24 +152,22 @@ jobs:
  - uses: actions/checkout@v2
 
  - name: Set up Singularity
- uses: eWaterCycle/setup-singularity@v6
+ uses: eWaterCycle/setup-singularity@v7
  with:
  singularity-version: ${{ env.singularity_version }}
 
  - name: Set up Python
  uses: actions/setup-python@v2
  with:
- python-version: 3.9
+ python-version: "3.10" # quoted, otherwise that turns into the number 3.1
+ cache: pip
+ cache-dependency-path: |
+ requirements.txt
+ test-requirements.txt
 
  - name: Give the test runner user a name to make provenance happy.
  run: sudo usermod -c 'CI Runner' $(whoami)
 
- - name: Cache for pip
- uses: actions/cache@v2
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-release-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
-
  - name: Install packages
  run: |
  pip install -U pip setuptools wheel

.github/workflows/codeql-analysis.yml

@@ -8,6 +8,10 @@ on:
  schedule:
  - cron: '0 10 * * 2'
 
+concurrency:
+ group: codeql-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
 jobs:
  CodeQL-Build:
 

.mergify.yml

@@ -1,13 +1,19 @@
+queue_rules:
+ - name: default
+ conditions:
+ - -draft # not a draft
+ - base=main
+
 pull_request_rules:
  - name: Automatic merge on approval and when when GitHub branch protection passes on main
  conditions:
  - "#approved-reviews-by>=1"
  - -draft
  - base=main
  actions:
- merge:
+ queue:
  method: merge
- strict: smart+fasttrack
+ name: default
 
 pull_request_rules:
  - name: Automatic merge for leadership team members when there are no reviewers and the label is "ready"
@@ -19,6 +25,6 @@ pull_request_rules:
  - author=@leadership
  - label=ready
  actions:
- merge:
+ queue:
  method: merge
- strict: smart+fasttrack
+ name: default

.readthedocs.yml

@@ -16,6 +16,6 @@ formats: all
 python:
  version: 3.7
  install:
+ - requirements: docs/requirements.txt
  - method: pip
  path: .
- extra_requirements: [ docs ]

CONTRIBUTING.md

@@ -1,21 +1,32 @@
 Style guide:
 - PEP-8
-- Python 3.6+ compatible code
+- Python 3.7+ compatible code
 - PEP-484 type hints
 - Vertically align the type hints in function definitions
 
-In order to contribute to the development of ``cwltool``, you need to install cwltool from source (preferably in a virtual environment):
+The development is done using ``git``, we encourage you to get familiar with it.
 Here's a rough guide (improvements are welcome!) 
-- Install virtualenv via pip: ``pip install virtualenv``
+To get the code and start working on the changes you can start a console and:
 - Clone the cwltool: ``git clone https://github.com/common-workflow-language/cwltool.git``
 - Switch to cwltool directory: ``cd cwltool``
-- Create a virtual environment: ``virtualenv cwltool``
+
+In order to contribute to the development of ``cwltool``, the source code needs to pass the test before your changes are accepted.
+There are a couple ways to test the code with your changes: let `tox` manage installation and test running in virtual environments, or do it manually (preferably in a virtual environment):
+- Install ``tox`` preferably using the OS' package manager, otherwise it can be installed with ``pip install --user -U tox``
+- Make your changes to the code and add tests for new cool things you're adding!
+- Run the tests with the command ``tox``, it's recommended to use some parameters as tox will try to run all the checks in all available python interpreters.
+- The important tests to run are ``unit tests`` and ``type tests``.
+ To run these two in Python 3.7, we can tell tox to run only those tests by running: ``tox -e py37-unit,py37-mypy2,py37-mypy3``.
+- Run ``tox -l`` to see all available tests and runtimes
+
+For the more traditional workflow:
+- Create a virtual environment: ``python3 -m venv cwltool``
 - To begin using the virtual environment, it needs to be activated: ``source bin/activate``
-- To check if you have the virtual environment set up: ``which python`` and it should point to python executable in your virtual env
-- Install cwltool in the virtual environment: ``pip install .``
+- To check if you have the virtual environment set up: ``which python`` and it should point to python executable in your virtualenv
+- Install cwltool: ``pip install -e .``
 - Check the version which might be different from the version installed in general on any system: ``cwltool --version``
-- After you've made the changes, you can the complete test suite via tox: ``tox``
-- If you want to run specific tests, say ``unit tests`` in Python 3.8, then: ``tox -e py38-unit``.
-- Look at ``tox.ini`` for all available tests and runtimes
-- If tests are passing, you can simply commit and create a PR on ``cwltool`` repo:
-- After you're done working on the ``cwltool``, you can deactivate the virtual environment: ``deactivate``
+- Make your changes to the code and add tests for new cool things you're adding!
+- Run the unit-tests to see : ``python setup.py test``
+- After you're done working on ``cwltool``, you can deactivate the virtual environment: ``deactivate``
+
+When tests are passing, you can simply commit and create a PR on ``cwltool`` repo

MANIFEST.in

@@ -1,16 +1,21 @@
-include README.rst
+include README.rst CODE_OF_CONDUCT.md CONTRIBUTING.md
 include MANIFEST.in
 include LICENSE.txt
-include *requirements.txt
+include *requirements.txt mypy.ini tox.ini
 include gittaggers.py Makefile cwltool.py
+recursive-include typeshed *.pyi
 include tests/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/tmp4/alpha/*
 include tests/wf/*
 include tests/wf/operation/*
 include tests/override/*
+include tests/reloc/*.cwl
+include tests/reloc/dir1/*
+include tests/reloc/dir2/*
 include tests/checker_wf/*
 include tests/subgraph/*
+include tests/input_deps/*
 include tests/trs/*
 include tests/wf/generator/*
 include cwltool/py.typed
@@ -61,9 +66,6 @@ prune cwltool/schemas/v1.0/salad/schema_salad/tests
 prune cwltool/schemas/v1.1.0-dev1/salad/typeshed
 prune cwltool/schemas/v1.1.0-dev1/salad/schema_salad/tests
 prune cwltool/schemas/presentations
-prune cwltool/schemas/draft-2
-prune cwltool/schemas/draft-1
-prune cwltool/schemas/draft-3
 prune cwltool/schemas/site
 prune cwltool/schemas/v1.0/examples
 prune cwltool/schemas/v1.0/v1.0

Makefile

@@ -25,9 +25,9 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \
-isort wheel autoflake flake8-bugbear pyupgrade bandit \
--rtest-requirements.txt -rmypy_requirements.txt
+DEVPKGS=diff_cover pylint pep257 pydocstyle tox tox-pyenv \
+isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\
+-rtest-requirements.txt -rmypy-requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
  python-flake8 python-mock shellcheck
 
@@ -50,6 +50,9 @@ install-dependencies: FORCE
 pip install --upgrade $(DEVPKGS)
 pip install -r requirements.txt
 
+install-doc-dep:
+pip install -r docs/requirements.txt
+
 ## install-deb-dep: install most of the dev dependencies via apt-get
 install-deb-dep:
 sudo apt-get install $(DEBDEVPKGS)
@@ -148,11 +151,11 @@ diff-cover.html: coverage.xml
 
 ## test : run the ${MODULE} test suite
 test: check-python3 $(PYSOURCES)
-python -m pytest ${PYTEST_EXTRA}
+python -m pytest -rs ${PYTEST_EXTRA}
 
 ## testcov : run the ${MODULE} test suite and collect coverage
 testcov: check-python3 $(PYSOURCES)
-python -m pytest --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
+python -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
 
 sloccount.sc: $(PYSOURCES) Makefile
 sloccount --duplicates --wide --details $^ > $@
@@ -177,14 +180,14 @@ mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES))
 
 mypyc: $(PYSOURCES)
 MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
- && pytest -vv ${PYTEST_EXTRA}
+ && pytest -rs -vv ${PYTEST_EXTRA}
 
 shellcheck: FORCE
 shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \
 cwltool-in-docker.sh
 
 pyupgrade: $(PYSOURCES)
-pyupgrade --exit-zero-even-if-changed --py36-plus $^
+pyupgrade --exit-zero-even-if-changed --py37-plus $^
 
 release-test: check-python3 FORCE
 git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )