diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 2b07e716f..8845e954a 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -108,13 +108,13 @@ jobs: df -h / # Display available space - name: Run tests - timeout-minutes: 10 + timeout-minutes: 30 env: AMI_NAME: "supabase-postgres-${{ steps.random.outputs.random_string }}" run: | # TODO: use poetry for pkg mgmt pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests - pytest -vv -s testinfra/test_ami_nix.py + pytest -vvvv -s testinfra/test_ami_nix.py - name: Cleanup resources on build cancellation if: ${{ cancelled() }} diff --git a/ansible/files/postgres_prestart.sh.j2 b/ansible/files/postgres_prestart.sh.j2 index 3ffe54c85..40e8debd7 100644 --- a/ansible/files/postgres_prestart.sh.j2 +++ b/ansible/files/postgres_prestart.sh.j2 @@ -26,7 +26,62 @@ update_orioledb_buffers() { fi } +check_extensions_file() { + local extensions_file="/root/pg_extensions.json" + if [ ! -f "$extensions_file" ]; then + echo "extensions: No extensions file found, skipping extensions versions check" + return 1 + fi + return 0 +} + +get_pg_cron_version() { + if ! check_extensions_file; then + return + fi + + local version + version=$(sudo -u postgres /var/lib/postgresql/.nix-profile/bin/jq -r '.pg_cron // empty' "/root/pg_extensions.json") + if [ -z "$version" ]; then + echo "pg_cron: Not specified in extensions file" + return + fi + + if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "pg_cron: Invalid version format: $version" + return + fi + + echo "$version" +} + +switch_pg_cron_version() { + local version="$1" + local switch_script="/var/lib/postgresql/.nix-profile/bin/switch_pg_cron_version" + + if [ ! -x "$switch_script" ]; then + echo "pg_cron: No version switch script available" + return + fi + + echo "pg_cron: Switching to version $version" + sudo -u postgres "$switch_script" "$version" + echo "pg_cron: Version switch completed" +} + +handle_pg_cron_version() { + local version + version=$(get_pg_cron_version) + if [ -n "$version" ]; then + switch_pg_cron_version "$version" + fi +} + main() { + # 1. pg_cron version handling + handle_pg_cron_version + + # 2. orioledb handling local has_orioledb=$(check_orioledb_enabled) if [ "$has_orioledb" -lt 1 ]; then return 0 diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 99b89d6d9..8b67eabe5 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -90,6 +90,12 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" when: stage2_nix + +- name: Install jq from nix binary cache + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#jq" + when: stage2_nix - name: Set ownership and permissions for /etc/ssl/private become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 0e658202d..6a4ee2f01 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.078-orioledb" - postgres17: "17.4.1.028" - postgres15: "15.8.1.085" + postgresorioledb-17: "17.0.1.067-orioledb-pgcron-4" + postgres17: "17.4.1.017-pgcron-4" + postgres15: "15.8.1.074-pgcron-4" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/flake.nix b/flake.nix index d49c9e3fa..6302a6894 100644 --- a/flake.nix +++ b/flake.nix @@ -637,157 +637,124 @@ --prefix PATH : ${pkgs.nushell}/bin ''; # Script to run the AMI build and tests locally - build-test-ami = pkgs.runCommand "build-test-ami" - { - buildInputs = with pkgs; [ - packer - awscli2 - yq - jq - openssl - git - coreutils - aws-vault - ]; - } '' - mkdir -p $out/bin - cat > $out/bin/build-test-ami << 'EOL' - #!/usr/bin/env bash - set -euo pipefail + build-test-ami = pkgs.writeScriptBin "build-test-ami" '' + #!/usr/bin/env bash + set -euo pipefail - show_help() { - cat << EOF - Usage: build-test-ami [--help] + show_help() { + cat << EOF + Usage: build-test-ami [--help] - Build AMI images for PostgreSQL testing. + Build AMI images for PostgreSQL testing. - This script will: - 1. Check for required tools and AWS authentication - 2. Build two AMI stages using Packer - 3. Clean up any temporary instances - 4. Output the final AMI name for use with run-testinfra + This script will: + 1. Check for required tools and AWS authentication + 2. Build two AMI stages using Packer + 3. Clean up any temporary instances + 4. Output the final AMI name for use with run-testinfra - Arguments: - postgres-version PostgreSQL major version to build (required) + Arguments: + postgres-version PostgreSQL major version to build (required) - Options: - --help Show this help message and exit + Options: + --help Show this help message and exit - Requirements: - - AWS Vault profile must be set in AWS_VAULT environment variable - - Packer, AWS CLI, yq, jq, and OpenSSL must be installed - - Must be run from a git repository + Requirements: + - AWS Vault profile must be set in AWS_VAULT environment variable + - Packer, AWS CLI, yq, jq, and OpenSSL must be installed + - Must be run from a git repository - Example: - aws-vault exec -- nix run .#build-test-ami 15 - EOF - } - - # Handle help flag - if [[ "$#" -gt 0 && "$1" == "--help" ]]; then - show_help - exit 0 - fi + Example: + aws-vault exec -- nix run .#build-test-ami 15 + EOF + } - export PATH="${pkgs.lib.makeBinPath (with pkgs; [ - packer - awscli2 - yq - jq - openssl - git - coreutils - aws-vault - ])}:$PATH" + # Handle help flag + if [[ "$#" -gt 0 && "$1" == "--help" ]]; then + show_help + exit 0 + fi - # Check for required tools - for cmd in packer aws-vault yq jq openssl; do - if ! command -v $cmd &> /dev/null; then - echo "Error: $cmd is required but not found" - exit 1 - fi - done + export PATH="${pkgs.lib.makeBinPath (with pkgs; [ + packer + awscli2 + yq + jq + openssl + git + coreutils + aws-vault + ])}:$PATH" - # Check AWS Vault profile - if [ -z "''${AWS_VAULT:-}" ]; then - echo "Error: AWS_VAULT environment variable must be set with the profile name" - echo "Usage: aws-vault exec -- nix run .#build-test-ami " + # Check for required tools + for cmd in packer aws-vault yq jq openssl; do + if ! command -v $cmd &> /dev/null; then + echo "Error: $cmd is required but not found" exit 1 fi + done - # Set values - REGION="ap-southeast-1" - POSTGRES_VERSION="$1" - RANDOM_STRING=$(openssl rand -hex 8) - GIT_SHA=$(git rev-parse HEAD) - RUN_ID=$(date +%s) - - # Generate common-nix.vars.pkr.hcl - PG_VERSION=$(yq -r ".postgres_release[\"postgres$POSTGRES_VERSION\"]" ansible/vars.yml) - echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - - # Build AMI Stage 1 - packer init amazon-arm64-nix.pkr.hcl - packer build \ - -var "git-head-version=$GIT_SHA" \ - -var "packer-execution-id=$RUN_ID" \ - -var-file="development-arm.vars.pkr.hcl" \ - -var-file="common-nix.vars.pkr.hcl" \ - -var "ansible_arguments=" \ - -var "postgres-version=$RANDOM_STRING" \ - -var "region=$REGION" \ - -var 'ami_regions=["'"$REGION"'"]' \ - -var "force-deregister=true" \ - -var "ansible_arguments=-e postgresql_major=$POSTGRES_VERSION" \ - amazon-arm64-nix.pkr.hcl - - # Build AMI Stage 2 - packer init stage2-nix-psql.pkr.hcl - packer build \ - -var "git-head-version=$GIT_SHA" \ - -var "packer-execution-id=$RUN_ID" \ - -var "postgres_major_version=$POSTGRES_VERSION" \ - -var-file="development-arm.vars.pkr.hcl" \ - -var-file="common-nix.vars.pkr.hcl" \ - -var "postgres-version=$RANDOM_STRING" \ - -var "region=$REGION" \ - -var 'ami_regions=["'"$REGION"'"]' \ - -var "force-deregister=true" \ - -var "git_sha=$GIT_SHA" \ - stage2-nix-psql.pkr.hcl - - # Cleanup instances from AMI builds - cleanup_instances() { - echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." - aws ec2 --region $REGION describe-instances \ - --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ - --query "Reservations[].Instances[].InstanceId" \ - --output text | xargs -r aws ec2 terminate-instances \ - --region $REGION --instance-ids || true - } - - # Set up traps for various signals to ensure cleanup - trap cleanup_instances EXIT HUP INT QUIT TERM - - # Create and activate virtual environment - VENV_DIR=$(mktemp -d) - trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM - python3 -m venv "$VENV_DIR" - source "$VENV_DIR/bin/activate" - - # Install required Python packages - echo "Installing required Python packages..." - pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests - - # Run the tests with aws-vault - echo "Running tests for AMI: $RANDOM_STRING using AWS Vault profile: $AWS_VAULT_PROFILE" - aws-vault exec $AWS_VAULT_PROFILE -- pytest -vv -s testinfra/test_ami_nix.py + # Check AWS Vault profile + if [ -z "''${AWS_VAULT:-}" ]; then + echo "Error: AWS_VAULT environment variable must be set with the profile name" + echo "Usage: aws-vault exec -- nix run .#build-test-ami " + exit 1 + fi - # Deactivate virtual environment (cleanup is handled by trap) - deactivate - EOL - chmod +x $out/bin/build-test-ami - ''; + # Set values + REGION="ap-southeast-1" + POSTGRES_VERSION="$1" + RANDOM_STRING=$(openssl rand -hex 8) + GIT_SHA=$(git rev-parse HEAD) + RUN_ID=$(date +%s) + + # Generate common-nix.vars.pkr.hcl + PG_VERSION=$(yq -r ".postgres_release[\"postgres$POSTGRES_VERSION\"]" ansible/vars.yml) + echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl + + # Build AMI Stage 1 + packer init amazon-arm64-nix.pkr.hcl + packer build \ + -var "git-head-version=$GIT_SHA" \ + -var "packer-execution-id=$RUN_ID" \ + -var-file="development-arm.vars.pkr.hcl" \ + -var-file="common-nix.vars.pkr.hcl" \ + -var "ansible_arguments=" \ + -var "postgres-version=$RANDOM_STRING" \ + -var "region=$REGION" \ + -var 'ami_regions=["'"$REGION"'"]' \ + -var "force-deregister=true" \ + -var "ansible_arguments=-e postgresql_major=$POSTGRES_VERSION" \ + amazon-arm64-nix.pkr.hcl + + # Build AMI Stage 2 + packer init stage2-nix-psql.pkr.hcl + packer build \ + -var "git-head-version=$GIT_SHA" \ + -var "packer-execution-id=$RUN_ID" \ + -var "postgres_major_version=$POSTGRES_VERSION" \ + -var-file="development-arm.vars.pkr.hcl" \ + -var-file="common-nix.vars.pkr.hcl" \ + -var "postgres-version=$RANDOM_STRING" \ + -var "region=$REGION" \ + -var 'ami_regions=["'"$REGION"'"]' \ + -var "force-deregister=true" \ + -var "git_sha=$GIT_SHA" \ + stage2-nix-psql.pkr.hcl + + # Cleanup instances from AMI builds + cleanup_instances() { + echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." + aws ec2 --region $REGION describe-instances \ + --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ + --query "Reservations[].Instances[].InstanceId" \ + --output text | xargs -r aws ec2 terminate-instances \ + --region $REGION --instance-ids || true + } + + # Set up traps for various signals to ensure cleanup + trap cleanup_instances EXIT HUP INT QUIT TERM + ''; run-testinfra = pkgs.runCommand "run-testinfra" { diff --git a/nix/ext/pg_cron-1.3.1-pg15.patch b/nix/ext/pg_cron-1.3.1-pg15.patch new file mode 100644 index 000000000..d3b6cd702 --- /dev/null +++ b/nix/ext/pg_cron-1.3.1-pg15.patch @@ -0,0 +1,31 @@ +diff --git a/src/pg_cron.c b/src/pg_cron.c +index e0ca973..4d51b2c 100644 +--- a/src/pg_cron.c ++++ b/src/pg_cron.c +@@ -14,6 +14,8 @@ + #include + + #include "postgres.h" ++#include "commands/async.h" ++#include "miscadmin.h" + #include "fmgr.h" + + /* these are always necessary for a bgworker */ +@@ -1908,7 +1910,7 @@ CronBackgroundWorker(Datum main_arg) + /* Post-execution cleanup. */ + disable_timeout(STATEMENT_TIMEOUT, false); + CommitTransactionCommand(); +- ProcessCompletedNotifies(); ++ /* ProcessCompletedNotifies removed */ + pgstat_report_activity(STATE_IDLE, command); + pgstat_report_stat(true); + +@@ -2025,7 +2027,7 @@ ExecuteSqlString(const char *sql) + */ + oldcontext = MemoryContextSwitchTo(parsecontext); + #if PG_VERSION_NUM >= 100000 +- querytree_list = pg_analyze_and_rewrite(parsetree, sql, NULL, 0,NULL); ++ querytree_list = pg_analyze_and_rewrite_fixedparams(parsetree, sql, NULL, 0, NULL); + #else + querytree_list = pg_analyze_and_rewrite(parsetree, sql, NULL, 0); + #endif diff --git a/nix/ext/pg_cron.nix b/nix/ext/pg_cron.nix index 792db7676..3f438931e 100644 --- a/nix/ext/pg_cron.nix +++ b/nix/ext/pg_cron.nix @@ -1,31 +1,140 @@ { lib, stdenv, fetchFromGitHub, postgresql }: -stdenv.mkDerivation rec { - pname = "pg_cron"; - version = "1.6.4"; +let + allVersions = { + "1.3.1" = { + rev = "v1.3.1"; + hash = "sha256-rXotNOtQNmA55ErNxGoNSKZ0pP1uxEVlDGITFHuqGG4="; + patches = [ ./pg_cron-1.3.1-pg15.patch ]; + }; + "1.4.2" = { + rev = "v1.4.2"; + hash = "sha256-P0Fd10Q1p+KrExb35G6otHpc6pD61WnMll45H2jkevM="; + }; + "1.6.4" = { + rev = "v1.6.4"; + hash = "sha256-t1DpFkPiSfdoGG2NgNT7g1lkvSooZoRoUrix6cBID40="; + }; + "1.5.2" = { + rev = "v1.5.2"; + hash = "sha256-+quVWbKJy6wXpL/zwTk5FF7sYwHA7I97WhWmPO/HSZ4="; + }; + }; + + # Simple version string that concatenates all versions with dashes + versionString = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings ["."] ["-"] v) (lib.attrNames allVersions)); + + mkPgCron = pgCronVersion: { rev, hash, patches ? [] }: stdenv.mkDerivation { + pname = "pg_cron"; + version = "${pgCronVersion}-pg${lib.versions.major postgresql.version}"; + + buildInputs = [ postgresql ]; + inherit patches; + + src = fetchFromGitHub { + owner = "citusdata"; + repo = "pg_cron"; + inherit rev hash; + }; - buildInputs = [ postgresql ]; + buildPhase = '' + make PG_CONFIG=${postgresql}/bin/pg_config + + # Create version-specific SQL file + cp pg_cron.sql pg_cron--${pgCronVersion}.sql - src = fetchFromGitHub { - owner = "citusdata"; - repo = pname; - rev = "v${version}"; - hash = "sha256-t1DpFkPiSfdoGG2NgNT7g1lkvSooZoRoUrix6cBID40="; + # Create versioned control file with modified module path + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/pg_cron'|" \ + pg_cron.control > pg_cron--${pgCronVersion}.control + ''; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension,bin} + + # Install versioned library + install -Dm755 pg_cron${postgresql.dlSuffix} $out/lib/pg_cron-${pgCronVersion}${postgresql.dlSuffix} + + # Install version-specific files + install -Dm644 pg_cron--${pgCronVersion}.sql $out/share/postgresql/extension/ + install -Dm644 pg_cron--${pgCronVersion}.control $out/share/postgresql/extension/ + + # Install upgrade scripts + find . -name 'pg_cron--*--*.sql' -exec install -Dm644 {} $out/share/postgresql/extension/ \; + ''; }; + getVersions = pg: + if lib.versionAtLeast pg.version "17" + then { "1.6.4" = allVersions."1.6.4"; } + else allVersions; + + allVersionsForPg = lib.mapAttrs mkPgCron (getVersions postgresql); + +in +stdenv.mkDerivation { + pname = "pg_cron-all"; + version = versionString; + + buildInputs = lib.attrValues allVersionsForPg; + + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + installPhase = '' - mkdir -p $out/{lib,share/postgresql/extension} + mkdir -p $out/{lib,share/postgresql/extension,bin} + + # Install all versions + for drv in ${lib.concatStringsSep " " (lib.attrValues allVersionsForPg)}; do + ln -sv $drv/lib/* $out/lib/ + cp -v --no-clobber $drv/share/postgresql/extension/* $out/share/postgresql/extension/ || true + done + + # Create default symlinks + latest_control=$(ls -v $out/share/postgresql/extension/pg_cron--*.control | tail -n1) + latest_version=$(basename "$latest_control" | sed -E 's/pg_cron--([0-9.]+).control/\1/') + + # Create main control file with default_version + echo "default_version = '$latest_version'" > $out/share/postgresql/extension/pg_cron.control + cat "$latest_control" >> $out/share/postgresql/extension/pg_cron.control + + # Library symlink + ln -sfnv pg_cron-$latest_version${postgresql.dlSuffix} $out/lib/pg_cron${postgresql.dlSuffix} + + # Create version switcher script + cat > $out/bin/switch_pg_cron_version <<'EOF' + #!/bin/sh + set -e + + if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo "Example: $0 1.4.2" + exit 1 + fi + + VERSION=$1 + LIB_DIR=$(dirname "$0")/../lib + + # Check if version exists + if [ ! -f "$LIB_DIR/pg_cron-$VERSION${postgresql.dlSuffix}" ]; then + echo "Error: Version $VERSION not found" + exit 1 + fi + + # Update library symlink + ln -sfnv "pg_cron-$VERSION${postgresql.dlSuffix}" "$LIB_DIR/pg_cron${postgresql.dlSuffix}" + + echo "Successfully switched pg_cron to version $VERSION" + EOF - cp *${postgresql.dlSuffix} $out/lib - cp *.sql $out/share/postgresql/extension - cp *.control $out/share/postgresql/extension + chmod +x $out/bin/switch_pg_cron_version ''; meta = with lib; { - description = "Run Cron jobs through PostgreSQL"; - homepage = "https://github.com/citusdata/pg_cron"; - changelog = "https://github.com/citusdata/pg_cron/raw/v${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + description = "Run Cron jobs through PostgreSQL (multi-version compatible)"; + homepage = "https://github.com/citusdata/pg_cron"; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; }; } diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 1975818d6..51278dfa4 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -162,6 +162,7 @@ "init_database_only": false }} """ +pg_cron_json = '{"pg_cron": "1.3.1"}' logger = logging.getLogger("ami-tests") handler = logging.StreamHandler() @@ -272,10 +273,11 @@ def gzip_then_base64_encode(s: str) -> str: - {{path: /etc/gotrue.env, content: {gzip_then_base64_encode(gotrue_env_content)}, permissions: '0664', encoding: gz+b64}} - {{path: /etc/wal-g/config.json, content: {gzip_then_base64_encode(walg_config_json_content)}, permissions: '0664', owner: 'wal-g:wal-g', encoding: gz+b64}} - {{path: /tmp/init.json, content: {gzip_then_base64_encode(init_json_content)}, permissions: '0600', encoding: gz+b64}} + - {{path: /root/pg_extensions.json, content: {gzip_then_base64_encode('{"pg_cron": "1.3.1"}')}, permissions: '0644', encoding: gz+b64}} runcmd: - 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt' - - 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .' - - 'bash init.sh "staging"' + - 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh . 2>&1 | tee /var/log/init-download.log' + - 'bash init.sh "staging" 2>&1 | tee /var/log/init-script.log' - 'touch /var/lib/init-complete' - 'rm -rf /tmp/*' """, @@ -342,6 +344,25 @@ def gzip_then_base64_encode(s: str) -> str: if attempt >= max_attempts: logger.error("init.sh failed to complete within the timeout period") + + # Check init script logs before terminating + try: + download_log = run_ssh_command(ssh, "sudo cat /var/log/init-download.log") + if download_log['succeeded']: + logger.error("Init script download log:") + logger.error(download_log['stdout']) + else: + logger.error(f"Failed to read download log: {download_log['stderr']}") + + init_log = run_ssh_command(ssh, "sudo cat /var/log/init-script.log") + if init_log['succeeded']: + logger.error("Init script execution log:") + logger.error(init_log['stdout']) + else: + logger.error(f"Failed to read init script log: {init_log['stderr']}") + except Exception as e: + logger.error(f"Error reading logs: {str(e)}") + instance.terminate() raise TimeoutError("init.sh failed to complete within the timeout period") @@ -355,22 +376,65 @@ def is_healthy(ssh) -> bool: ("fail2ban", "sudo fail2ban-client status"), ] + service_status = {} for service, command in health_checks: try: result = run_ssh_command(ssh, command) if not result['succeeded']: logger.warning(f"{service} not ready") - return False - except Exception: - logger.warning(f"Connection failed during {service} check") - return False - + logger.error(f"{service} command failed") + logger.error(f"{service} stdout: {result['stdout']}") + logger.error(f"{service} stderr: {result['stderr']}") + + # Always read and log the PostgreSQL logs + logger.warning("PostgreSQL status check:") + try: + log_files = [ + "/var/log/postgresql/*.log", + "/var/log/postgresql/*.csv" + ] + + for log_pattern in log_files: + log_result = run_ssh_command(ssh, f"sudo cat {log_pattern}") + if log_result['succeeded']: + logger.error(f"PostgreSQL logs from {log_pattern}:") + logger.error(log_result['stdout']) + if log_result['stderr']: + logger.error(f"Log read errors: {log_result['stderr']}") + else: + logger.error(f"Failed to read PostgreSQL logs from {log_pattern}: {log_result['stderr']}") + except Exception as e: + logger.error(f"Error reading PostgreSQL logs: {str(e)}") + + service_status[service] = False + else: + service_status[service] = True + + except Exception as e: + logger.warning(f"Connection failed during {service} check, attempting reconnect...") + logger.error(f"Error details: {str(e)}") + service_status[service] = False + + # Log overall status of all services + logger.info("Service health status:") + for service, healthy in service_status.items(): + logger.info(f"{service}: {'healthy' if healthy else 'unhealthy'}") + + # If any service is unhealthy, wait and return False with status + if not all(service_status.values()): + if service_status.get("postgres", False): # If postgres is healthy but others aren't + sleep(5) # Only wait if postgres is up but other services aren't + logger.warning("Some services are not healthy, will retry...") + return False + + logger.info("All services are healthy, proceeding to tests...") return True while True: if is_healthy(ssh): break - sleep(1) + logger.warning("Health check failed, retrying...") + sleep(5) # Return both the SSH connection and instance IP for use in tests yield { @@ -507,3 +571,32 @@ def test_postgrest_ending_empty_key_query_parameter_is_removed(host): }, ) assert res.ok + + +def test_pg_cron_extension(host): + # Only run this test for PostgreSQL 15 + postgres_version = os.environ.get("POSTGRES_MAJOR_VERSION") + if postgres_version != "15": + pytest.skip(f"Skipping pg_cron test for PostgreSQL version {postgres_version}") + + # Connect as supabase_admin and create the extension + with host.sudo("postgres"): + result = host.run('psql -U supabase_admin -d postgres -c "CREATE EXTENSION pg_cron WITH SCHEMA pg_catalog VERSION \'1.3.1\';"') + assert result.rc == 0, f"Failed to create pg_cron extension: {result.stderr}" + + # Create test table + result = host.run('psql -U supabase_admin -d postgres -c "CREATE TABLE cron_test_log (id SERIAL PRIMARY KEY, message TEXT, log_time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);"') + assert result.rc == 0, f"Failed to create test table: {result.stderr}" + + # Schedule a job + result = host.run('psql -U supabase_admin -d postgres -c "SELECT cron.schedule(\'* * * * *\', \'INSERT INTO cron_test_log (message) VALUES (\\\'Hello from pg_cron!\\\');\');"') + assert result.rc == 0, f"Failed to schedule job: {result.stderr}" + assert "1" in result.stdout, "Expected schedule ID 1" + + # Verify job is scheduled + result = host.run('psql -U supabase_admin -d postgres -c "SELECT * FROM cron.job;"') + assert result.rc == 0, f"Failed to query cron.job: {result.stderr}" + assert "* * * * *" in result.stdout, "Expected cron schedule pattern" + assert "INSERT INTO cron_test_log" in result.stdout, "Expected cron command" + assert "postgres" in result.stdout, "Expected postgres username" + assert "postgres" in result.stdout, "Expected postgres database"