Use cibuildwheel to build release wheels

Author: elprans

.github/workflows/build-manylinux-wheels.sh

@@ -3,14 +3,42 @@
 set -Eexuo pipefail
 shopt -s nullglob
 
-export DEBIAN_FRONTEND=noninteractive
+PGVERSION=${PGVERSION:-12}
 
-apt-get install -y --no-install-recommends curl
-curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-mkdir -p /etc/apt/sources.list.d/
-echo "deb https://apt.postgresql.org/pub/repos/apt/ ${DISTRO_NAME}-pgdg main" \
- >> /etc/apt/sources.list.d/pgdg.list
-apt-get update
-apt-get install -y --no-install-recommends \
- postgresql-${PGVERSION} \
- postgresql-contrib-${PGVERSION}
+if [ -e /etc/os-release ]; then
+ source /etc/os-release
+elif [ -e /etc/centos-release ]; then
+ ID="centos"
+ VERSION_ID=$(cat /etc/centos-release | cut -f3 -d' ' | cut -f1 -d.)
+else
+ echo "install-postgres.sh: cannot determine which Linux distro this is" >&2
+ exit 1
+fi
+
+if [ "${ID}" = "debian" -o "${ID}" = "ubuntu" ]; then
+ export DEBIAN_FRONTEND=noninteractive
+
+ apt-get install -y --no-install-recommends curl gnupg ca-certificates
+ curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+ mkdir -p /etc/apt/sources.list.d/
+ echo "deb https://apt.postgresql.org/pub/repos/apt/ ${VERSION_CODENAME}-pgdg main" \
+ >> /etc/apt/sources.list.d/pgdg.list
+ apt-get update
+ apt-get install -y --no-install-recommends \
+ "postgresql-${PGVERSION}" \
+ "postgresql-contrib-${PGVERSION}"
+elif [ "${ID}" = "centos" ]; then
+ el="EL-${VERSION_ID}-$(arch)"
+ baseurl="https://download.postgresql.org/pub/repos/yum/reporpms"
+ yum install -y "${baseurl}/${el}/pgdg-redhat-repo-latest.noarch.rpm"
+ if [ ${VERSION_ID} -ge 8 ]; then
+ dnf -qy module disable postgresql
+ fi
+ yum install -y \
+ "postgresql${PGVERSION}-server" \
+ "postgresql${PGVERSION}-contrib"
+ ln -s "/usr/pgsql-${PGVERSION}/bin/pg_config" "/usr/local/bin/pg_config"
+else
+ echo "install-postgres.sh: Unsupported distro: ${distro}" >&2
+ exit 1
+fi

.github/workflows/install-postgres.sh

@@ -66,21 +66,16 @@ jobs:
  - uses: actions/upload-artifact@v2
  with:
  name: dist
- path: dist/
+ path: dist/*.tar.*
 
  build-wheels:
  needs: validate-release-request
  runs-on: ${{ matrix.os }}
  strategy:
  matrix:
- python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0.rc.1]
  os: [ubuntu-latest, macos-latest, windows-latest]
- arch: [x86_64]
- exclude:
- - os: windows-latest
- arch: aarch64
- - os: macos-latest
- arch: aarch64
+ cibw_python: ["cp37-*", "cp38-*", "cp39-*", "cp310-*"]
+ cibw_arch: ["auto64"]
 
  defaults:
  run:
@@ -95,121 +90,29 @@ jobs:
  fetch-depth: 50
  submodules: true
 
- - name: Set up QEMU
- if: matrix.arch == 'aarch64'
- uses: docker/setup-qemu-action@v1
-
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python-version }}
-
- - name: Install Python Deps
- run: |
- python -m pip install --upgrade setuptools pip wheel
-
- - name: Build Wheels (linux)
- if: startsWith(matrix.os, 'ubuntu')
+ - uses: pypa/cibuildwheel@v2.1.1
  env:
- PYTHON_VERSION: ${{ matrix.python-version }}
- ARCH: ${{ matrix.arch }}
- run: |
- case "${ARCH}" in
- x86_64)
- mlimg=manylinux1_x86_64
- ;;
- aarch64)
- mlimg=manylinux2014_aarch64
- ;;
- *)
- echo "Unsupported wheel arch: ${ARCH}" >&2
- exit 1
- ;;
- esac
-
- docker run --rm \
- -v "${GITHUB_WORKSPACE}":/github/workspace:rw \
- --workdir=/github/workspace \
- -e GITHUB_WORKSPACE=/github/workspace \
- -e PYTHON_VERSION="${PYTHON_VERSION}" \
- --entrypoint=/github/workspace/.github/workflows/build-manylinux-wheels.sh \
- quay.io/pypa/${mlimg}
-
- - name: Build Wheels (non-linux)
- if: "!startsWith(matrix.os, 'ubuntu')"
- run: |
- make clean
- python setup.py bdist_wheel
-
- - name: Test Wheels (native)
- if: |
- !contains(github.event.pull_request.labels.*.name, 'skip wheel tests')
- && matrix.arch == 'x86_64'
- env:
- OS: ${{ matrix.os }}
- run: |
- if [ "${OS}" = "windows-latest" ]; then
- export PGINSTALLATION="${PGBIN}"
- fi
- "${GITHUB_WORKSPACE}/.github/workflows/test-wheels.sh"
-
- - name: Test Wheels (emulated)
- if: |
- !contains(github.event.pull_request.labels.*.name, 'skip wheel tests')
- && matrix.arch != 'x86_64'
- env:
- PYTHON_VERSION: ${{ matrix.python-version }}
- PGVERSION: 13
- DISTRO_NAME: focal
- ARCH: ${{ matrix.arch }}
- run: |
- sudo env DISTRO_NAME="${DISTRO_NAME}" PGVERSION="${PGVERSION}" \
- .github/workflows/install-postgres.sh
- # Allow docker guest to connect to the database
- echo "port = 5433" | \
- sudo tee --append /etc/postgresql/${PGVERSION}/main/postgresql.conf
- echo "listen_addresses = '*'" | \
- sudo tee --append /etc/postgresql/${PGVERSION}/main/postgresql.conf
- echo "host all all 172.17.0.0/16 trust" | \
- sudo tee --append /etc/postgresql/${PGVERSION}/main/pg_hba.conf
- if [ "${PGVERSION}" -ge "11" ]; then
- # Disable JIT to avoid unpredictable timings in tests.
- echo "jit = off" | \
- sudo tee --append /etc/postgresql/${PGVERSION}/main/postgresql.conf
- fi
- sudo pg_ctlcluster ${PGVERSION} main restart
-
- case "${ARCH}" in
- aarch64)
- img="docker.io/arm64v8/python:${PYTHON_VERSION}-buster"
- ;;
- *)
- echo "Unsupported wheel arch: ${ARCH}" >&2
- exit 1
- ;;
- esac
-
- docker run --rm \
- -v "${GITHUB_WORKSPACE}":/github/workspace:rw \
- -e GITHUB_WORKSPACE=/github/workspace \
- -e PYTHON_VERSION="${PYTHON_VERSION}" \
- --workdir=/github/workspace/ \
- ${img} \
- /bin/bash -ex -c ' \
- echo GITHUB_WORKSPACE=${GITHUB_WORKSPACE} >> /etc/environment \
- && echo PGHOST=$(ip route | grep default | cut -f3 -d" " | uniq) \
- >> /etc/environment \
- && echo PGPORT=5433 >> /etc/environment \
- && echo PGUSER=postgres >> /etc/environment \
- && echo ENVIRON_FILE /etc/environment >> /etc/login.defs \
- && useradd -m -s /bin/bash test \
- && su -l test /github/workspace/.github/workflows/test-wheels.sh \
- '
+ CIBW_BUILD_VERBOSITY: 1
+ CIBW_BUILD: ${{ matrix.cibw_python }}
+ CIBW_ARCHS: ${{ matrix.cibw_arch }}
+ CIBW_BEFORE_ALL_LINUX: >
+ yum -y install libffi-devel
+ && env PGVERSION=12 .github/workflows/install-postgres.sh
+ && useradd -m -s /bin/bash apgtest
+ CIBW_TEST_EXTRAS: "test"
+ CIBW_TEST_COMMAND: >
+ python {project}/tests/__init__.py
+ CIBW_TEST_COMMAND_WINDOWS: >
+ python {project}\tests\__init__.py
+ CIBW_TEST_COMMAND_LINUX: >
+ PY=`which python`
+ && chmod -R go+rX "$(dirname $(dirname $(dirname $PY)))"
+ && su -p -l apgtest -c "$PY {project}/tests/__init__.py"
 
  - uses: actions/upload-artifact@v2
  with:
  name: dist
- path: dist/
+ path: wheelhouse/*.whl
 
  publish-docs:
  needs: validate-release-request

.github/workflows/release.yml

@@ -59,11 +59,6 @@ jobs:
  with:
  python-version: ${{ matrix.python-version }}
 
- - name: Configure PostgreSQL Environment
- if: matrix.os == 'windows-latest'
- run: |
- echo PGINSTALLATION="${PGBIN}" >> "${GITHUB_ENV}"
-
  - name: Install Python Deps
  if: steps.release.outputs.version == 0
  run: |

.github/workflows/test-wheels.sh

@@ -53,7 +53,10 @@ class Cluster:
  def __init__(self, data_dir, *, pg_config_path=None):
  self._data_dir = data_dir
  self._pg_config_path = pg_config_path
- self._pg_bin_dir = os.environ.get('PGINSTALLATION')
+ self._pg_bin_dir = (
+ os.environ.get('PGINSTALLATION')
+ or os.environ.get('PGBIN')
+ )
  self._pg_ctl = None
  self._daemon_pid = None
  self._daemon_process = None
@@ -518,7 +521,10 @@ def _run_pg_config(self, pg_config_path):
 
  def _find_pg_config(self, pg_config_path):
  if pg_config_path is None:
- pg_install = os.environ.get('PGINSTALLATION')
+ pg_install = (
+ os.environ.get('PGINSTALLATION')
+ or os.environ.get('PGBIN')
+ )
  if pg_install:
  pg_config_path = platform_exe(
  os.path.join(pg_install, 'pg_config'))