chore: build pg15 with docker

.github/workflows/ami-release.yml

@@ -56,6 +56,25 @@ jobs:
  tar xvf "$layer" -C ansible/files/extensions --strip-components 1
  done
 
+ - name: Build Postgres deb
+ uses: docker/build-push-action@v3
+ with:
+ push: false
+ load: true
+ file: docker/Dockerfile
+ target: pg-deb
+ tags: supabase/postgres:deb
+ platforms: linux/${{ matrix.arch }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+ - name: Extract Postgres deb
+ run: |
+ mkdir -p /tmp/build ansible/files/postgres
+ docker save supabase/postgres:deb | tar xv -C /tmp/build
+ for layer in /tmp/build/*/layer.tar; do
+ tar xvf "$layer" -C ansible/files/postgres --strip-components 1
+ done
+
  - name: Build AMI
  run: |
  GIT_SHA=${{github.sha}}

ansible/tasks/setup-postgres.yml

@@ -1,70 +1,13 @@
-# Downloading dependencies
-- name: GPG dependencies
- apt:
- name: gnupg
-
-- name: Add Postgres GPG key
- apt_key:
- keyserver: keyserver.ubuntu.com
- id: B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
-
-- name: Add Postgres PPA
- apt_repository:
- repo: "deb https://apt-archive.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg-archive main"
- state: present
+- name: Postgres - copy package
+ copy:
+ src: files/postgres/
+ dest: /tmp/build/
 
-# let's build binaries from their published source packages
-- name: Add Postgres PPA - source
+- name: Postgres - add PPA
  apt_repository:
- repo: "deb-src https://apt-archive.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg-archive main"
+ repo: "deb [ trusted=yes ] file:///tmp/build ./"
  state: present
 
-- name: Create temporary build directory
- tempfile:
- state: directory
- register: pg_build_dir
-
-- name: Setting mcpu (arm)
- set_fact:
- mcpu: "neoverse-n1"
- when: platform == "arm64"
-
-- name: Postgres - build
- shell: |
- set -e
- export PYTHONDONTWRITEBYTECODE=1
- savedAptMark="$(apt-mark showmanual)"
- cd "{{ pg_build_dir.path }}"
-
- # create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be)
- apt-get update
- apt-get install -y --no-install-recommends dpkg-dev
- echo "deb [ trusted=yes ] file://{{ pg_build_dir.path }} ./" > /etc/apt/sources.list.d/temp.list
- _update_repo() {
- dpkg-scanpackages . > Packages
- apt-get -o Acquire::GzipIndexes=false update
- }
- _update_repo
-
- # build .deb files from upstream's source packages (which are verified by apt-get)
- export DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)"
- export DEB_CPPFLAGS_APPEND="-mcpu={{ mcpu }} -fsigned-char"
-
- # we have to build postgresql-common first because postgresql-15 shares "debian/rules" logic with it: https://salsa.debian.org/postgresql/postgresql/-/commit/99f44476e258cae6bf9e919219fa2c5414fa2876
- # (and it "Depends: pgdg-keyring")
- apt-get build-dep -y postgresql-common pgdg-keyring
- apt-get source --compile postgresql-common pgdg-keyring
- _update_repo
- apt-get build-dep -y "postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1"
- apt-get source --compile "postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1"
-
- # we don't remove APT lists here because they get re-downloaded and removed later
- # reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies
- # (which is done after we install the built packages so we don't have to redownload any overlapping dependencies)
- apt-mark showmanual | xargs apt-mark auto > /dev/null
- apt-mark manual $savedAptMark
- _update_repo
-
 - name: Postgres - install commons
  apt:
  name: postgresql-common
@@ -79,17 +22,15 @@
  name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1
  install_recommends: no
 
-# if we have leftovers from building, let's purge them (including extra, unnecessary build deps)
-- name: Remove build dependencies
- shell: |
- set -e
- rm -rf /var/lib/apt/lists/*
- apt-get purge -y --auto-remove
- rm -rf "{{ pg_build_dir.path }}" /etc/apt/sources.list.d/temp.list
- find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +
-
-- name: Hold postgres {{ postgresql_release }} from apt upgrade
- shell: apt-mark hold postgresql-{{ postgresql_major }}
+- name: Postgres - remove PPA
+ apt_repository:
+ repo: "deb [ trusted=yes ] file:///tmp/build ./"
+ state: absent
+
+- name: Postgres - cleanup package
+ file:
+ path: /tmp/build
+ state: absent
 
 - name: Create symlink to /usr/lib/postgresql/bin
  shell:

common.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.1.0.89-rc0"
+postgres-version = "15.1.0.89-rc1"

docker/Dockerfile

@@ -0,0 +1,71 @@
+ARG ubuntu_release=focal
+FROM ubuntu:${ubuntu_release} as base
+
+ARG ubuntu_release
+ARG postgresql_major=15
+ARG postgresql_release=${postgresql_major}.1
+
+FROM base as pg-source
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ gnupg \
+ dpkg-dev \
+ ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# Add Postgres PPA
+ARG postgresql_gpg_key=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${postgresql_gpg_key}" && \
+ echo "deb https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list && \
+ echo "deb-src https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list
+
+# Create local PPA
+WORKDIR /tmp/build
+RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \
+ dpkg-scanpackages . > Packages && \
+ apt-get -o Acquire::GzipIndexes=false update
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)"
+
+# Configure processor optimised build
+ARG CPPFLAGS="-mcpu=neoverse-n1"
+ENV DEB_CPPFLAGS_APPEND="${CPPFLAGS} -fsigned-char"
+
+RUN apt-get build-dep -y postgresql-common pgdg-keyring && \
+ apt-get source --compile postgresql-common pgdg-keyring && \
+ dpkg-scanpackages . > Packages && \
+ apt-get -o Acquire::GzipIndexes=false update
+
+RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+ apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+ dpkg-scanpackages . > Packages && \
+ apt-get -o Acquire::GzipIndexes=false update
+
+# Remove source directories
+RUN rm -rf /tmp/build/*/
+
+FROM base as pg
+
+# Inherit args from base stage
+ARG postgresql_major
+ARG postgresql_release
+
+COPY --from=pg-source /tmp/build /tmp/build
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \
+ apt-get -o Acquire::GzipIndexes=false update && \
+ apt-get install -y --no-install-recommends postgresql-common && \
+ sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf && \
+ apt-get install -y --no-install-recommends "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+ rm -rf /var/lib/apt/lists/* && \
+ rm -rf /tmp/build /etc/apt/sources.list.d/temp.list
+
+ENV PATH $PATH:/usr/lib/postgresql/${postgresql_major}/bin
+
+FROM scratch as pg-deb
+
+COPY --from=pg-source /tmp/build /tmp