chore: consolidate to just one start-client for postgres tooling (#1194)

* chore: consolidate to just one start-client for postgres tooling * fix: needed session_preload_libraries = 'supautils' in conf * chore: rm debug items --------- Co-authored-by: Sam Rose <[email protected]>
supabase · Sep 13, 2024 · 6e17f0b · 6e17f0b
1 parent dd15fb6
commit 6e17f0b
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 91 deletions.
diff --git a/flake.nix b/flake.nix
@@ -342,19 +342,24 @@
                 name = "pg_ident.conf";
                 path = ./ansible/files/postgresql_config/pg_ident.conf.j2;
               };
+              postgresqlExtensionCustomScriptsPath = builtins.path {
+                name = "extension-custom-scripts";
+                path = ./ansible/files/postgresql_extension_custom_scripts;
+              };
               getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
               localeArchive = if pkgs.stdenv.isDarwin
                 then "${pkgs.darwin.locale}/share/locale"
                 else "${pkgs.glibcLocales}/lib/locale/locale-archive";
             in
             pkgs.runCommand "start-postgres-server" { } ''
-              mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql
+              mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts
               cp ${supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; }
               cp ${pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; }
               cp ${loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; }
               cp ${readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; }
               cp ${pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; }
               cp ${pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; }
+              cp -r ${postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; }
               echo "Copy operation completed"
               chmod 644 $out/etc/postgresql-custom/supautils.conf
               chmod 644 $out/etc/postgresql/postgresql.conf
@@ -371,40 +376,31 @@
                 --subst-var-by 'SUPAUTILS_CONF_FILE' "$out/etc/postgresql-custom/supautils.conf" \
                 --subst-var-by 'PG_HBA' "$out/etc/postgresql/pg_hba.conf" \
                 --subst-var-by 'PG_IDENT' "$out/etc/postgresql/pg_ident.conf" \
-                --subst-var-by 'LOCALES' '${localeArchive}'
+                --subst-var-by 'LOCALES' '${localeArchive}' \
+                --subst-var-by 'EXTENSION_CUSTOM_SCRIPTS_DIR' "$out/extension-custom-scripts"
                 
               chmod +x $out/bin/start-postgres-server
             '';
 
-          # Start a version of the client.
-          start-client = pkgs.runCommand "start-postgres-client" { } ''
-            mkdir -p $out/bin
-            substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \
-              --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
-              --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
-              --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'
-            chmod +x $out/bin/start-postgres-client
-          '';
-
           # Start a version of the client and runs migrations script on server.
-          start-client-and-migrate =
+          start-client =
             let
               migrationsDir = ./migrations/db;
               postgresqlSchemaSql = ./nix/tools/postgresql_schema.sql;
               pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
               statExtensionSql = ./ansible/files/stat_extension.sql;
             in
-            pkgs.runCommand "start-postgres-client-migrate" { } ''
+            pkgs.runCommand "start-postgres-client" { } ''
               mkdir -p $out/bin
-              substitute ${./nix/tools/run-client-migrate.sh.in} $out/bin/start-postgres-client-migrate \
+              substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \
                 --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
                 --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
                 --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
                 --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \
                 --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \
                 --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \
                 --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}'
-              chmod +x $out/bin/start-postgres-client-migrate
+              chmod +x $out/bin/start-postgres-client
             '';
 
           # Migrate between two data directories.
@@ -559,7 +555,6 @@
           {
             start-server = mkApp "start-server" "start-postgres-server";
             start-client = mkApp "start-client" "start-postgres-client";
-            start-client-and-migrate = mkApp "start-client-and-migrate" "start-postgres-client-migrate";
             start-replica = mkApp "start-replica" "start-postgres-replica";
             migration-test = mkApp "migrate-tool" "migrate-postgres";
             sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions";

diff --git a/nix/tools/run-client-migrate.sh.in b/nix/tools/run-client-migrate.sh.in
diff --git a/nix/tools/run-client.sh.in b/nix/tools/run-client.sh.in
@@ -20,9 +20,35 @@ else
     echo "Please provide a valid Postgres version (15, 16, or orioledb-16)"
     exit 1
 fi
-
+#vars for migration.sh
 export PATH=$BINDIR/bin:$PATH
-
+export POSTGRES_DB=postgres
+export POSTGRES_HOST=localhost
+export POSTGRES_PORT=@PGSQL_DEFAULT_PORT@
 PORTNO="${2:-@PGSQL_DEFAULT_PORT@}"
+PGSQL_SUPERUSER=@PGSQL_SUPERUSER@
+MIGRATIONS_DIR=@MIGRATIONS_DIR@
+POSTGRESQL_SCHEMA_SQL=@POSTGRESQL_SCHEMA_SQL@
+PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@
+STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@
+psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL
+    create role postgres superuser login password '$PGPASSWORD';
+    alter database postgres owner to postgres;
+EOSQL
+for sql in "$MIGRATIONS_DIR"/init-scripts/*.sql; do
+    echo "$0: running $sql"
+    psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -f  "$sql" postgres
+done
+psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"
+psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL"
+psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL"
+# run migrations as super user - postgres user demoted in post-setup
+for sql in "$MIGRATIONS_DIR"/migrations/*.sql; do
+    echo "$0: running $sql"
+    psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -f "$sql" postgres
+done
+psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -f "$POSTGRESQL_SCHEMA_SQL" postgres
+# TODO Do we need to reset stats when running migrations locally?
+#psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -c 'SELECT extensions.pg_stat_statements_reset(); SELECT pg_stat_reset();' postgres || true
 
 exec psql -U postgres -p "$PORTNO" -h localhost postgres
diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 # shellcheck shell=bash
-
 [ ! -z "$DEBUG" ] && set -x
-
 # first argument should be '15' or '16' for the version
 if [ "$1" == "15" ]; then
     echo "Starting server for PSQL 15"
@@ -20,9 +18,7 @@ else
     echo "Please provide a valid Postgres version (15, 16 or orioledb-16)"
     exit 1
 fi
-
 export PATH=$BINDIR/bin:$PATH
-
 PGSQL_SUPERUSER=@PGSQL_SUPERUSER@
 PSQL_CONF_FILE=@PSQL_CONF_FILE@
 PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@
@@ -32,6 +28,7 @@ LOGGING_CONFIG_FILE=@LOGGING_CONF_FILE@
 READREPL_CONFIG_FILE=@READREPL_CONF_FILE@
 PG_HBA_FILE=@PG_HBA@
 PG_IDENT_FILE=@PG_IDENT@
+EXTENSION_CUSTOM_SCRIPTS=@EXTENSION_CUSTOM_SCRIPTS_DIR@
 DATDIR=$(mktemp -d)
 LOCALE_ARCHIVE=@LOCALES@
 export LOCALE_ARCHIVE
@@ -41,22 +38,26 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_CTYPE=en_US.UTF-8
 mkdir -p "$DATDIR"
-
 echo "NOTE: using port $PORTNO for server"
 echo "NOTE: using temporary directory $DATDIR for data, which will not be removed"
 echo "NOTE: you are free to re-use this data directory at will"
-
 initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" 
-
 echo "NOTE: patching postgresql.conf files"
-
-sed -e "\$a\\
-    include = '$SUPAUTILS_CONFIG_FILE' \\
-    pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \
-    -e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \
-    -e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$PG_HBA_FILE'|" \
-    -e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$PG_IDENT_FILE'|" \
-    -e "s|include = '/etc/postgresql/logging.conf'|#&|" \
-    -e "s|include = '/etc/postgresql-custom/read-replica.conf'|include = '$READREPL_CONFIG_FILE'|" \
-    "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf"
-postgres -p "$PORTNO" -D "$DATDIR" -k /tmp 
+cp "$PG_HBA_FILE" "$DATDIR/pg_hba.conf"
+cp "$PG_IDENT_FILE" "$DATDIR/pg_ident.conf"
+cp "$READREPL_CONFIG_FILE" "$DATDIR/read-replica.conf"
+cp -r "$EXTENSION_CUSTOM_SCRIPTS" "$DATDIR"
+sed "s|supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'|supautils.privileged_extensions_custom_scripts_path = '$DATDIR/extension-custom-scripts'|" "$SUPAUTILS_CONFIG_FILE" > "$DATDIR/supautils.conf"
+sed -e "1i\\
+include = '$DATDIR/supautils.conf'" \
+-e "\$a\\
+pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \
+-e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \
+-e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$DATDIR/pg_hba.conf'|" \
+-e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$DATDIR/pg_ident.conf'|" \
+-e "s|include = '/etc/postgresql/logging.conf'|#&|" \
+-e "s|include = '/etc/postgresql-custom/read-replica.conf'|include = '$DATDIR/read-replica.conf'|" \
+-e "\$a\\
+session_preload_libraries = 'supautils'" \
+"$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf"
+postgres --config-file="$DATDIR/postgresql.conf" -p "$PORTNO" -D "$DATDIR" -k /tmp