ENG-7737 | MFR convert to Poetry

Author: sh-andriy

.github/workflows/test-build.yml

@@ -26,13 +26,12 @@ jobs:
  uses: actions/cache@v4
  with:
  path: ~/.cache/pip
- key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
+ key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
 
  - name: Install dependencies
  run: |
- python -m pip install --upgrade pip==24.0
- pip install setuptools==80.1.0
- pip install -r dev-requirements.txt
+ python -m pip install poetry==2.1.2 setuptools==80.1.0
+ poetry install --no-root --without=docs --with=dev
 
  runtests:
  name: Run unit tests
@@ -55,25 +54,25 @@ jobs:
  uses: actions/cache@v4
  with:
  path: ~/.cache/pip
- key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
+ key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
 
  - name: Install test dependencies
  run: |
- python -m pip install --upgrade pip==24.0
- pip install setuptools==80.1.0
- pip install -r dev-requirements.txt
+ python -m pip install poetry==2.1.2
+ python -m pip install setuptools==80.1.0
+ poetry install --no-root --without=docs --with=dev
 
  - name: Run flake8
- run: flake8 .
+ run: poetry run flake8 .
 
  - name: Build plugins
- run: python setup.py develop
+ run: poetry run python setup.py develop
 
  - name: Run unit tests
  run: |
- pytest --cov-report term-missing --cov modular-file-renderer tests
+ poetry run pytest --cov-report term-missing --cov mfr tests
 
  - name: Upload coverage to Coveralls
- run: coveralls --service=github
+ run: poetry run coveralls --service=github
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -43,21 +43,22 @@ RUN usermod -d /home www-data \
 RUN mkdir -p /code
 WORKDIR /code
 
-RUN pip install -U pip==24.0
-RUN pip install setuptools==69.5.1
-RUN pip install unoconv==0.9.0
+COPY pyproject.toml poetry.lock* /code/
 
-COPY ./requirements.txt /code/
+ENV POETRY_NO_INTERACTION=1 \
+ POETRY_VIRTUALENVS_CREATE=0 \
+ POETRY_VIRTUALENVS_IN_PROJECT=1
 
-RUN pip install --no-cache-dir -r ./requirements.txt
+RUN pip install poetry==2.1.2 setuptools==80.1.0 \
+ && poetry install --no-root --without=docs
 
 # Copy the rest of the code over
 COPY ./ /code/
 
 ARG GIT_COMMIT=
 ENV GIT_COMMIT=${GIT_COMMIT}
 
-RUN python setup.py develop
+RUN poetry run python setup.py develop
 
 EXPOSE 7778
 

constraints.txt

@@ -0,0 +1,74 @@
+[tool.poetry]
+name = "mfr"
+version = "25.0.0"
+description = "Modular File Renderer"
+authors = ["Center for Open Science <contact@cos.io>"]
+readme = "README.md"
+# requires-python = ">=3.13,<3.14"
+license = "Apache License 2.0"
+
+[tool.poetry.dependencies]
+python = "^3.13"
+aiohttp = "3.10.5"
+chardet = "5.2.0"
+furl = "2.0.0"
+humanfriendly = "10.0"
+invoke = "2.2.0"
+mako = "1.0.1"
+sentry-sdk = "2.22.0"
+setuptools = "80.1.0"
+stevedore = "5.4.1"
+tornado = "6.4.2"
+agent = "0.1.2"
+google-auth = "2.38.0"
+Pygments = "2.19.1"
+pydocx = "0.9.10"
+olefile = "0.44"
+Pillow = "11.0.0"
+psd-tools = "1.4"
+ipython = "7.31.1"
+nbconvert = "4.2.0"
+nbformat = "4.1.0"
+traitlets = "4.2.2"
+jsonschema = "2.4.0"
+jinja2 = "3.1.3"
+mistune = "0.8.1"
+reportlab = "4.4.0"
+docutils = "0.21.2"
+pandas = "2.2.3"
+xlrd = "1.0.0"
+h5py = "3.13"
+scipy = "1.14.1"
+markdown = "3.8.0"
+certifi = "2025.1.31"
+unoconv = "0.9.0"
+
+waterbutler = { git = "https://github.com/CenterForOpenScience/waterbutler.git", branch = "feature/buff-worms" }
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+beautifulsoup4 = "4.13.3"
+colorlog = "6.9.0"
+coveralls = "3.3.1"
+flake8 = "7.1.2"
+ipdb = "0.13.13"
+mccabe = "0.7.0"
+pydevd = "3.3.0"
+pyflakes = "3.2.0"
+pytest = "8.1.1"
+pytest-cov = "4.1.0"
+pyzmq = "26.2.1"
+
+[tool.poetry.group.docs]
+optional = true
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "8.2.3"
+sphinx-autobuild = "2024.10.3"
+sphinx-rtd-theme = "3.0.2"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

dev-requirements.txt

@@ -1,13 +1,6 @@
 from setuptools import setup, find_namespace_packages
 
 
-def parse_requirements(requirements_txt):
- with open(requirements_txt) as f:
- return [l.strip('\n') for l in f if l.strip('\n') and not l.startswith('#')]
-
-
-requirements = parse_requirements('requirements.txt')
-
 # Taken from option 3 of https://packaging.python.org/guides/single-sourcing-package-version/
 version = {}
 with open('mfr/version.py') as fp:
@@ -28,7 +21,7 @@ def parse_requirements(requirements_txt):
  'Natural Language :: English',
  'Intended Audience :: Developers',
  'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.13',
  'Development Status :: 5 - Production/Stable',
  'License :: OSI Approved :: Apache Software License',
  ],

doc-requirements.txt

@@ -3,30 +3,24 @@
 from invoke import task
 
 WHEELHOUSE_PATH = os.environ.get('WHEELHOUSE')
-CONSTRAINTS_FILE = 'constraints.txt'
 
 
 @task
-def wheelhouse(ctx, develop=False):
- req_file = 'dev-requirements.txt' if develop else 'requirements.txt'
- cmd = f'pip wheel --find-links={WHEELHOUSE_PATH} -r {req_file} --wheel-dir={WHEELHOUSE_PATH} -c {CONSTRAINTS_FILE}'
- ctx.run(cmd, pty=True)
+def wheelhouse(ctx, develop=False, pty=True):
+ extras = '--with dev' if develop else ''
+ cmd = f'poetry export --format=requirements.txt {extras} | pip wheel --find-links={WHEELHOUSE_PATH} -r /dev/stdin --wheel-dir={WHEELHOUSE_PATH}'
+ ctx.run(cmd, pty=pty)
 
 
 @task
-def install(ctx, develop=False):
- ctx.run('python setup.py develop')
- req_file = 'dev-requirements.txt' if develop else 'requirements.txt'
- cmd = f'pip install --upgrade -r {req_file} -c {CONSTRAINTS_FILE}'
-
- if WHEELHOUSE_PATH:
- cmd += f' --no-index --find-links={WHEELHOUSE_PATH}'
- ctx.run(cmd, pty=True)
+def install(ctx, develop=False, pty=True):
+ extras = '--with dev' if develop else ''
+ ctx.run(f'poetry install {extras}', pty=pty)
 
 
 @task
 def flake(ctx):
- ctx.run('flake8 .', pty=True)
+ ctx.run('poetry run flake8 .', pty=True)
 
 
 @task
@@ -50,7 +44,7 @@ def test(ctx, verbose=False, nocov=False, extension=None, path=None):
  path = ''
  coverage = ' --cov-report term-missing --cov mfr' if not nocov else ''
  verbose = '-v' if verbose else ''
- cmd = f'py.test{coverage} tests{path} {verbose}'
+ cmd = f'poetry run pytest{coverage} tests{path} {verbose}'
  ctx.run(cmd, pty=True)