chore: mv ansible migration test to nix flake check

migration tests are now run on `nix flake check`: - tests for some extensions are adjusted to run conditionally on non-orioledb + preserves behavior from the ansible tests - ebssurrogate/files/unit-tests/unit-test-01.sql are deleted + these tests are already included nix/test/prime.sql cleanup ansible/tasks/test-image.yml: - remove pg_prove since it's no longer used - remove unit tests since the same logic is already included nix/tests/prime.sql - remove migration tests since it's now tested with nix flake check

Author: steve-chavez

amazon-arm64-nix.pkr.hcl

@@ -228,11 +228,6 @@ build {
  destination = "/tmp"
  }
 
- provisioner "file" {
- source = "ebssurrogate/files/unit-tests"
- destination = "/tmp"
- }
-
  # Copy ansible playbook
  provisioner "shell" {
  inline = ["mkdir /tmp/ansible-playbook"]

ansible/tasks/test-image.yml

@@ -1,9 +1,3 @@
-- name: install pg_prove
- apt:
- pkg:
- - libtap-parser-sourcehandler-pgtap-perl
- when: debpkg_mode
-
 # - name: Temporarily disable PG Sodium references in config
 # become: yes
 # become_user: postgres
@@ -16,9 +10,9 @@
  become_user: postgres
  shell:
  cmd: >
- sed -i.bak 
- -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' 
- -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' 
+ sed -i.bak
+ -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+ -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/'
  -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
  -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
  /etc/postgresql/postgresql.conf
@@ -74,53 +68,6 @@
  LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
  when: stage2_nix
 
-
-- name: Check psql_version and modify migrations if oriole-xx
- block:
- - name: Check if psql_version is psql_orioledb-xx
- set_fact:
- is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
-
- - name: Remove specified extensions from SQL file
- ansible.builtin.command:
- cmd: >
- sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql
- when: is_psql_oriole
- become: yes
-
- - name: Remove specified extension files from extensions directory
- ansible.builtin.find:
- paths: /tmp/migrations/tests/extensions
- patterns: 
- - '*timescaledb*.sql'
- - '*plv8*.sql'
- - '*postgis*.sql'
- - '*pgrouting*.sql'
- register: files_to_remove
- when: is_psql_oriole
-
- - name: Delete matched extension files
- ansible.builtin.file:
- path: "{{ item.path }}"
- state: absent
- loop: "{{ files_to_remove.files }}"
- when: is_psql_oriole
- become: yes
-
-- name: Run Unit tests (with filename unit-test-*) on Postgres Database
- shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql
- register: retval
- failed_when: retval.rc != 0
- when: debpkg_mode or stage2_nix
-
-- name: Run migrations tests
- shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql
- register: retval
- failed_when: retval.rc != 0
- when: debpkg_mode or stage2_nix
- args:
- chdir: /tmp/migrations
-
 - name: Re-enable PG Sodium references in config
  become: yes
  become_user: postgres
@@ -132,14 +79,6 @@
  shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();'
  when: debpkg_mode or stage2_nix
 
-- name: remove pg_prove
- apt:
- pkg:
- - libtap-parser-sourcehandler-pgtap-perl
- state: absent
- autoremove: yes
- when: debpkg_mode
-
 - name: Stop Postgres Database
  become: yes
  become_user: postgres

ebssurrogate/files/unit-tests/unit-test-01.sql

@@ -180,9 +180,6 @@ function setup_chroot_environment {
 # Copy migrations
 cp -r /tmp/migrations /mnt/tmp/
 
-# Copy unit tests 
-cp -r /tmp/unit-tests /mnt/tmp/
-
 # Copy the bootstrap script into place and execute inside chroot
 cp /tmp/chroot-bootstrap-nix.sh /mnt/tmp/chroot-bootstrap-nix.sh
 chroot /mnt /tmp/chroot-bootstrap-nix.sh

ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -807,6 +807,9 @@
  exit 1
  fi
 
+ echo "Running migrations tests"
+ pg_prove -p 5435 -U supabase_admin -h localhost -d postgres -v ${./migrations/tests}/test.sql
+
  mkdir -p $out/regression_output
  if ! pg_regress \
  --use-existing \

flake.nix

@@ -1,43 +1,38 @@
-BEGIN;
-create extension if not exists postgis_sfcgal with schema "extensions" cascade;
-ROLLBACK;
+begin;
+do $_$
+begin
+ if not exists (select 1 from pg_extension where extname = 'orioledb') then
+ -- create postgis tiger as supabase_admin
+ create extension if not exists postgis_tiger_geocoder cascade;
 
-BEGIN;
-create extension if not exists postgis_raster with schema "extensions" cascade;
-ROLLBACK;
+ -- \ir ansible/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql
+ grant usage on schema tiger, tiger_data to postgres with grant option;
+ grant all privileges on all tables in schema tiger, tiger_data to postgres with grant option;
+ grant all privileges on all routines in schema tiger, tiger_data to postgres with grant option;
+ grant all privileges on all sequences in schema tiger, tiger_data to postgres with grant option;
+ alter default privileges in schema tiger, tiger_data grant all on tables to postgres with grant option;
+ alter default privileges in schema tiger, tiger_data grant all on routines to postgres with grant option;
+ alter default privileges in schema tiger, tiger_data grant all on sequences to postgres with grant option;
+ SET search_path TO extensions, public, tiger, tiger_data;
+ -- postgres role should have access
+ set local role postgres;
+ perform tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
 
-BEGIN;
--- create postgis tiger as supabase_admin
-create extension if not exists address_standardizer with schema extensions;
-create extension if not exists postgis_tiger_geocoder cascade;
+ -- other roles can be granted access
+ grant usage on schema tiger, tiger_data to authenticated;
+ grant select on all tables in schema tiger, tiger_data to authenticated;
+ grant execute on all routines in schema tiger, tiger_data to authenticated;
 
--- \ir ansible/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql
-grant usage on schema tiger, tiger_data to postgres with grant option;
-grant all privileges on all tables in schema tiger, tiger_data to postgres with grant option;
-grant all privileges on all routines in schema tiger, tiger_data to postgres with grant option;
-grant all privileges on all sequences in schema tiger, tiger_data to postgres with grant option;
-alter default privileges in schema tiger, tiger_data grant all on tables to postgres with grant option;
-alter default privileges in schema tiger, tiger_data grant all on routines to postgres with grant option;
-alter default privileges in schema tiger, tiger_data grant all on sequences to postgres with grant option;
-SET search_path TO extensions, public, tiger, tiger_data;
--- postgres role should have access
-set local role postgres;
-select tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
+ -- authenticated role should have access now
+ set local role authenticated;
+ perform tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
+ reset role;
 
--- other roles can be granted access
-grant usage on schema tiger, tiger_data to authenticated;
-grant select on all tables in schema tiger, tiger_data to authenticated;
-grant execute on all routines in schema tiger, tiger_data to authenticated;
-
--- authenticated role should have access now
-set local role authenticated;
-select tiger.pprint_addy(tiger.pagc_normalize_address('710 E Ben White Blvd, Austin, TX 78704'));
-ROLLBACK;
-
-BEGIN;
--- address standardizer creates a table in extensions schema, owned by supabase_admin
-create extension if not exists address_standardizer_data_us with schema extensions;
--- postgres role should have access
-set local role postgres;
-select * from extensions.us_lex;
-ROLLBACK;
+ -- postgres role should have access to address_standardizer_data_us
+ set local role postgres;
+ perform 1 from us_lex;
+ reset role;
+ end if;
+end
+$_$;
+rollback;

migrations/tests/extensions/01-postgis.sql

@@ -1,3 +1,9 @@
-BEGIN;
-create extension if not exists pgrouting with schema "extensions" cascade;
-ROLLBACK;
+begin;
+do $_$
+begin
+ if not exists (select 1 from pg_extension where extname = 'orioledb') then
+ create extension if not exists pgrouting with schema "extensions" cascade;
+ end if;
+end
+$_$;
+rollback;

migrations/tests/extensions/02-pgrouting.sql

@@ -1,3 +1,9 @@
-BEGIN;
-create extension if not exists timescaledb with schema "extensions";
-ROLLBACK;
+begin;
+do $_$
+begin
+ if not exists (select 1 from pg_extension where extname = 'orioledb') then
+ create extension if not exists timescaledb with schema "extensions";
+ end if;
+end
+$_$;
+rollback;

migrations/tests/extensions/10-timescaledb.sql

@@ -1,3 +1,9 @@
-BEGIN;
-create extension if not exists plv8 with schema "pg_catalog";
-ROLLBACK;
+begin;
+do $_$
+begin
+ if not exists (select 1 from pg_extension where extname = 'orioledb') then
+ create extension if not exists plv8 with schema "pg_catalog";
+ end if;
+end
+$_$;
+rollback;

migrations/tests/extensions/13-plv8.sql

@@ -122,11 +122,6 @@ build {
  destination = "/tmp"
  }
 
- provisioner "file" {
- source = "ebssurrogate/files/unit-tests"
- destination = "/tmp"
- }
-
  provisioner "shell" {
  environment_vars = [
  "POSTGRES_MAJOR_VERSION=${var.postgres-major-version}",