fix: we need to produce a series of migration files on pkg build

.github/workflows/test.yml

@@ -23,6 +23,10 @@ jobs:
       POSTGRES_PASSWORD: password
     steps:
       - uses: actions/checkout@v3
+      - id: settings
+        # Remove spaces and quotes to get the raw version string
+        run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT
+
       - id: args
         uses: mikefarah/yq@master
         with:
@@ -36,23 +40,23 @@ jobs:
         with:
           load: true
           context: .
+          file: "Dockerfile-156"
           target: production
           build-args: |
             ${{ steps.args.outputs.result }}
-          tags: supabase/postgres:latest
+          tags: supabase/postgres:${{ steps.settings.outputs.postgres-version }}
           cache-from: |
-            type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
-            type=gha,scope=${{ github.base_ref }}-latest-${{ matrix.arch }}
-          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
+            type=gha,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
+            type=gha,scope=${{ github.base_ref }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
 
       - name: Start Postgres
         run: |
           docker run --rm --pull=never \
           -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \
           -p ${{ env.POSTGRES_PORT }}:5432 \
           --name supabase_postgres \
-          -d supabase/postgres:latest
-
+          -d supabase/postgres:${{ steps.settings.outputs.postgres-version }}
       - name: Install psql
         run: |
           sudo apt update

ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh

@@ -171,6 +171,50 @@ EOF
     done
 }
 
+function patch_wrappers {
+    local IS_NIX_UPGRADE=$1
+
+    # This is a workaround for older versions of wrappers which don't have the expected
+    #  naming scheme, containing the version in their library's file name
+    #  e.g. wrappers-0.1.16.so, rather than wrappers.so
+    # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's
+    #  library directory, so we're making sure the new version has the expected (old version's)
+    #  file name.
+    # After the upgrade completes, the new version's library file is used.
+    # i.e. 
+    #  - old version: wrappers-0.1.16.so
+    #  - new version: wrappers-0.1.18.so
+    #  - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so
+    if [ "$IS_NIX_UPGRADE" = "true" ]; then
+        OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
+        OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
+
+        find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do
+            if [ -f "$WRAPPERS_LIB_PATH" ]; then
+                WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH")
+                if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then
+                    echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}"
+                    cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}"
+                fi
+            fi
+        done
+    else
+        if [ -d "$PGLIBOLD" ]; then
+            WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit)
+            if [ -f "$WRAPPERS_LIB_PATH" ]; then
+                OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
+                if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then
+                    LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
+                    if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then
+                        echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}"
+                        cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}"
+                    fi
+                fi
+            fi
+        fi
+    fi
+}
+
 function initiate_upgrade {
     mkdir -p "$MOUNT_POINT"
     SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/")
@@ -324,30 +368,7 @@ function initiate_upgrade {
         export LD_LIBRARY_PATH="${PGLIBNEW}"
     fi
 
-    # This is a workaround for older versions of wrappers which don't have the expected
-    #  naming scheme, containing the version in their library's file name
-    #  e.g. wrappers-0.1.16.so, rather than wrappers.so
-    # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's
-    #  library directory, so we're making sure the new version has the expected (old version's)
-    #  file name.
-    # After the upgrade completes, the new version's library file is used.
-    # i.e. 
-    #  - old version: wrappers-0.1.16.so
-    #  - new version: wrappers-0.1.18.so
-    #  - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so
-    if [ -d "$PGLIBOLD" ]; then
-        WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit)
-        if [ -f "$WRAPPERS_LIB_PATH" ]; then
-            OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
-            if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then
-                LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
-                if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then
-                    echo "Copying $OLD_WRAPPER_LIB_PATH to $WRAPPERS_LIB_PATH"
-                    cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}"
-                fi
-            fi
-        fi
-    fi
+    patch_wrappers "$IS_NIX_UPGRADE"
 
     echo "9. Creating new data directory, initializing database"
     chown -R postgres:postgres "$MOUNT_POINT/"

ansible/files/postgresql_config/postgresql.conf.j2

@@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english'
 #local_preload_libraries = ''
 #session_preload_libraries = ''
 
-shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter'	# (change requires restart)
+shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace'	# (change requires restart)
 jit_provider = 'llvmjit'		# JIT library to use
 
 # - Other Defaults -

ansible/playbook.yml

@@ -206,3 +206,9 @@
       shell: |
         sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove osquery"
       when: stage2_nix
+
+    - name: nix collect garbage
+      become: yes
+      shell: |
+        sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d"
+      when: stage2_nix

ansible/vars.yml

@@ -141,11 +141,13 @@ hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c
 pg_repack_release: "1.5.0"
 pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52
 
-pgvector_release: "0.7.0"
-pgvector_release_checksum: sha256:1b5503a35c265408b6eb282621c5e1e75f7801afc04eecb950796cfee2e3d1d8
+pgvector_release: "0.7.4"
+pgvector_release_checksum: sha256:0341edf89b1924ae0d552f617e14fb7f8867c0194ed775bcc44fa40288642583
 
 pg_tle_release: "1.3.2"
 pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1
 
 index_advisor_release: "0.2.0"
 index_advisor_checksum: sha256:2d3642012a9185cda51f1e82ba43d64a81b24a2655a3ac3afdcbbd95d46a1a27
+
+pg_backtrace_release: "1.1"

common-nix.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.6.1.104"
+postgres-version = "15.6.1.109"

common.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.1.1.86"
+postgres-version = "15.1.1.90"

flake.nix

@@ -61,6 +61,7 @@
         };
 
         sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
+        pg_regress = pkgs.callPackage ./nix/ext/pg_regress.nix { };
 
         # Our list of PostgreSQL extensions which come from upstream Nixpkgs.
         # These are maintained upstream and can easily be used here just by
@@ -101,6 +102,7 @@
           ./nix/ext/postgis.nix
           ./nix/ext/pgrouting.nix
           ./nix/ext/pgtap.nix
+          ./nix/ext/pg_backtrace.nix
           ./nix/ext/pg_cron.nix
           ./nix/ext/pgsql-http.nix
           ./nix/ext/pg_plan_filter.nix
@@ -269,6 +271,7 @@
           #psql_16 = makePostgres "16";
           #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16;
           sfcgal = sfcgal;
+          pg_regress = pg_regress;
           pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
           # Start a version of the server.
           start-server =
@@ -367,7 +370,7 @@
           in
           pkgs.runCommand "postgres-${pgpkg.version}-check-harness"
             {
-              nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove procps ];
+              nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove pg_regress procps ];
             } ''
             TMPDIR=$(mktemp -d)
             if [ $? -ne 0 ]; then
@@ -415,7 +418,19 @@
             done
             createdb -p 5432 -h localhost testing
             psql -p 5432 -h localhost -d testing -Xaf ${./nix/tests/prime.sql}
+
             pg_prove -p 5432 -h localhost -d testing ${sqlTests}/*.sql
+
+            mkdir -p $out/regression_output
+            pg_regress \
+              --use-existing \
+              --dbname=testing \
+              --inputdir=${./nix/tests} \
+              --outputdir=$out/regression_output \
+              --host=localhost \
+              --port=5432 \
+              $(ls ${./nix/tests/sql} | sed -e 's/\..*$//' | sort )
+
             pg_ctl -D "$PGDATA" stop
             mv $TMPDIR/logfile/postgresql.log $out
             echo ${pgpkg}

nix/docs/adding-tests.md

@@ -1,12 +1,46 @@
 There are basically two types of tests you can add:
 
 - pgTAP based tests, and
+- pg\_regress tests
 - Migration tests.
 
-In both cases, a number of extensions may be installed into the database for
+In all cases, a number of extensions may be installed into the database for
 use; you can see those in both [postgresql.conf.in](../tests/postgresql.conf.in)
 and [prime.sql](../tests/prime.sql) (extensions may be enabled in either place.)
 
+## pg\_regress tests
+
+pg\_regress tests are in [tests/sql](./../tests/sql/) with output in [tests/expected](./../tests/expected/).
+To create a new test, create a new SQL file in [tests/sql](./../tests/sql/) and then run:
+
+```
+nix flake check -L
+```
+
+Next, review the logs to identify where the test output was written
+
+```
+postgres> CREATE EXTENSION IF NOT EXISTS index_advisor;
+postgres> CREATE EXTENSION  
+postgres> (using postmaster on localhost, port 5432)    
+postgres> ============== running regression test queries        ==============
+postgres> test new_test                     ... diff: /nix/store/5gk419ddz7mzzwhc9j6yj5i8lkw67pdl-tests/expected/new_test.out: No such file or directory
+postgres> diff command failed with status 512: diff  "/nix/store/5gk419ddz7mzzwhc9j6yj5i8lkw67pdl-tests/expected/new_test.out" "/nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/regression_output/results/new_test.out" > "/nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/regression_output/results/new_test.out.diff
+```
+
+and copy the `regression_output` directory to where you can review
+
+```
+cp -r /nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/regression_output .
+```
+
+Then you can review the contents of `regression_output/results/new_test.out` to see if it matches what you expected.
+
+If it does match your expectations, copy the file to [tests/expected](./../tests/expected/) and the test will pass on the next run.
+
+If the output does not match your expectations, update the `<new_test>.sql` file, re-run with `nix flake check -L` and try again
+
+
 ## pgTAP tests
 
 These are super easy: simply add `.sql` files to the

nix/ext/pg_backtrace.nix

@@ -0,0 +1,33 @@
+{ lib, stdenv, fetchFromGitHub, postgresql }:
+
+stdenv.mkDerivation rec {
+  pname = "pg_backtrace";
+  version = "1.1";
+
+  buildInputs = [ postgresql ];
+
+  src = fetchFromGitHub {
+    owner  = "pashkinelfe";
+    repo   = pname;
+    rev    = "d100bac815a7365e199263f5b3741baf71b14c70";
+    hash = "sha256-IVCL4r4oj1Ams03D8y+XCFkckPFER/W9tQ68GkWQQMY=";
+  };
+
+  makeFlags = [ "USE_PGXS=1" ];
+
+  installPhase = ''
+    mkdir -p $out/{lib,share/postgresql/extension}
+
+    cp *.so      $out/lib
+    cp *.sql     $out/share/postgresql/extension
+    cp *.control $out/share/postgresql/extension
+  '';
+
+  meta = with lib; {
+    description = "Updated fork of pg_backtrace";
+    homepage    = "https://github.com/pashkinelfe/pg_backtrace";
+    maintainers = with maintainers; [ samrose ];
+    platforms   = postgresql.meta.platforms;
+    license     = licenses.postgresql;
+  };
+}

nix/ext/pg_regress.nix

@@ -0,0 +1,24 @@
+{ lib
+, stdenv
+, postgresql
+}:
+
+stdenv.mkDerivation {
+  pname = "pg_regress";
+  version = postgresql.version;
+
+  phases = [ "installPhase" ];
+
+  installPhase = ''
+    mkdir -p $out/bin
+    cp ${postgresql}/lib/pgxs/src/test/regress/pg_regress $out/bin/
+  '';
+
+  meta = with lib; {
+    description = "Regression testing tool for PostgreSQL";
+    homepage = "https://www.postgresql.org/";
+    maintainers = with maintainers; [ samrose ];
+    platforms = postgresql.meta.platforms;
+    license = licenses.postgresql;
+  };
+}

nix/ext/pgvector.nix

@@ -2,15 +2,15 @@
 
 stdenv.mkDerivation rec {
   pname = "pgvector";
-  version = "0.7.0";
+  version = "0.7.4";
 
   buildInputs = [ postgresql ];
 
   src = fetchFromGitHub {
     owner = "pgvector";
     repo = pname;
     rev = "refs/tags/v${version}";
-    hash = "sha256-vFn7sNphOYyig6Jl1HILMaC2t9strFQBQ8ywL8Ibx1M=";
+    hash = "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI=";
   };
 
   installPhase = ''