Merge pull request #458 from metaplex-foundation/release/v0.6.0

Author: StanChe

.env.example

@@ -1,46 +1,63 @@
+DOCKER_IMAGE_TAG=develop-latest
+
 RPC_HOST='https://mainnet-aura.metaplex.com/{personal_rpc_key}'
 
+#Configurable app parts that could be enabled or disabled. (values in the default positions)
+CHECK_PROOFS=false
+ENABLE_ROCKS_MIGRATION=true
+RESTORE_ROCKS_DB=false
+RUN_API=true
+RUN_BACKFILLER=true
+RUN_BUBBLEGUM_BACKFILLER=true
+RUN_GAPFILLER=false
+RUN_PROFILING=false
+SHOULD_REINGEST=false
+SKIP_CHECK_TREE_GAPS=true
+
 #Postgres
 PG_DATABASE_URL='postgres://solana:solana@localhost:5432/aura_db'
+POSTGRE_DB_PATH="/postgre/db/path"
 
 #Redis
 REDIS_CONNECTION_CONFIG='{"redis_connection_str":"redis://127.0.0.1:6379/0"}'
 
 #RocksDB
 ROCKS_DB_PATH="/usr/src/rocksdb-data"
-ROCKS_DB_SECONDARY_PATH="path/to/rocks/secondary/db"
+ROCKS_DB_SECONDARY_PATH="/path/to/rocks/secondary/db"
 # path to the slots data, required for the backfiller to work
 ROCKS_SLOTS_DB_PATH=/path/to/slots-data
-ROCKS_SECONDARY_SLOTS_DB_PATH=/path/to/secondary/ingester-slots
-ROCKS_ARCHIVES_DIR="path/to/rocks/backup/archives"
-ROCKS_BACKUP_ARCHIVES_DIR="path/to/rocks/backup/archives"
+
+ROCKS_ARCHIVES_DIR="/path/to/rocks/backup/archives"
+ROCKS_BACKUP_ARCHIVES_DIR="/path/to/rocks/backup/archives"
+ROCKS_BACKUP_DIR="/path/to/rocks/backup/"
 ROCKS_BACKUP_URL="127.0.0.1:3051/snapshot"
+ROCKS_DUMP_PATH="/path/to/dump"
 ROCKS_MIGRATION_STORAGE_PATH=/path/to/migration_storage
+ROCKS_SECONDARY_SLOTS_DB_PATH=/path/to/secondary/ingester-slots
 
 #Backfiller
 BACKFILLER_SOURCE_MODE=bigtable
-BIG_TABLE_CONFIG='{creds="/usr/src/app/creds.json", timeout=1000}'
+BIG_TABLE_CONFIG='{"creds": "/usr/src/app/creds.json", "timeout": 1000}'
+
+# Synchronizer
+DUMP_SYNCHRONIZER_BATCH_SIZE=10000
+
+# API server port (if API is enabled)
+API_SERVER_PORT=8990
+INGESTER_SERVER_PORT=9092
 
 # Metrics port. Start HTTP server to report metrics if port exist.
 API_METRICS_PORT=8985
 INGESTER_METRICS_PORT=9091
 MIGRATOR_METRICS_PORT=5091
 SYNCHRONIZER_METRICS_PORT=6091
 
-# API server port (if API is enabled)
-INGESTER_SERVER_PORT=9092
-API_SERVER_PORT=8990
-
 # GRPC Server port
 PEER_GRPC_PORT=9099
+GAPFILLER_PEER_ADDR="0.0.0.0"
 
-# Docker
-# Required by Postgre container
-POSTGRE_DB_PATH="postgre/db/path"
-ROCKS_BACKUP_DIR="path/to/rocks/backup/"
-FILE_STORAGE_PATH="path/to/file/storage"
+FILE_STORAGE_PATH="/path/to/file/storage"
 FILE_STORAGE_PATH_CONTAINER="/usr/src/app/file_storage"
-ROCKS_DUMP_PATH="/path/to/dump"
 
 #Profiling (optional)
 PROFILING_FILE_PATH_CONTAINER="/usr/src/profiling"
@@ -60,70 +77,3 @@ INTEGRITY_VERIFICATION_TEST_FILE_PATH="./test_keys/test_keys.txt"
 INTEGRITY_VERIFICATION_TEST_FILE_PATH_CONTAINER="/test_keys/test_keys.txt"
 INTEGRITY_VERIFICATION_SLOTS_COLLECT_PATH="./slots_collect"
 INTEGRITY_VERIFICATION_SLOTS_COLLECT_PATH_CONTAINER="/slots_collect"
-
-#Configurable app parts that cold be enabled or disabled. (values in the default positions)
-RUN_API=true
-RUN_BACKFILLER=true
-RUN_BUBBLEGUM_BACKFILLER=true
-RUN_GAPFILLER=false
-SHOULD_REINGEST=false
-RUN_PROFILING=false
-RESTORE_ROCKS_DB=false
-ENABLE_ROCKS_MIGRATION=true
-CHECK_PROOFS=false
-SKIP_CHECK_TREE_GAPS=false
-
-#Changes (todo Remove after review/migration)
-
-# API_RPC_HOST  INGESTER_RPC_HOST -> RPC_HOST
-#INGESTER_ROCKS_DB_PATH -> ROCKS_DB_PATH
-#INGESTER_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
-#INGESTER_SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
-#INGESTER_FILE_STORAGE_PATH -> FILE_STORAGE_PATH
-#INGESTER_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
-# INGESTER_PROFILING_FILE_PATH -> PROFILING_FILE_PATH
-# INGESTER_PROFILING_FILE_PATH_CONTAINER -> PROFILING_FILE_PATH_CONTAINER
-#  INGESTER_MIGRATION_STORAGE_PATH -> ROCKS_MIGRATION_STORAGE_PATH
-#
-# INGESTER_ROCKS_BACKUP_ARCHIVES_DIR -> ROCKS_BACKUP_ARCHIVES_DIR
-# INGESTER_ROCKS_BACKUP_DIR ->  ROCKS_BACKUP_DIR
-#
-#SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#INGESTER_SLOTS_DB_PATH -> ROCKS_SLOTS_DB_PATH
-#INGESTER_SECONDARY_SLOTS_DB_PATH -> ROCKS_SECONDARY_SLOTS_DB_PATH
-
-#API_DATABASE_CONFIG ->  PG_MAX_DB_CONNECTIONS  and  PG_DATABASE_URL
-
-#SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#SYNCHRONIZER_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH
-#SYNCHRONIZER_DUMP_SYNCHRONIZER_BATCH_SIZE -> DUMP_SYNCHRONIZER_BATCH_SIZE
-#SYNCHRONIZER_DUMP_SYNC_THRESHOLD -> DUMP_SYNC_THRESHOLD
-
-#API_ARCHIVES_DIR -> rocks_archives_dir
-#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
-#API_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH
-#API_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
-#API_JSON_MIDDLEWARE_CONFIG -> JSON_MIDDLEWARE_CONFIG
-#API_CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD - > CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD
-#API_CONSISTENCE_BACKFILLING_SLOTS_THRESHOLD -> CONSISTENCE_BACKFILLING_SLOTS_THRESHOLD
-#
-#API_SKIP_CHECK_TREE_GAPS -> SKIP_CHECK_TREE_GAPS
-# INGESTER_REDIS_MESSENGER_CONFIG -> REDIS_CONNECTION_CONFIG and note the differnce (less complex type)
-# INGESTER_BACKFILLER_SOURCE_MODE -> BACKFILLER_SOURCE_MODE
-# INGESTER_BIG_TABLE_CONFIG -> BIG_TABLE_CONFIG
-# both API_PEER_GRPC_PORT and INGESTER_PEER_GRPC_PORT were replaced with a single PEER_GRPC_PORT - only one will actually work
-# API_PEER_GRPC_MAX_GAP_SLOTS and INGESTER_PEER_GRPC_MAX_GAP_SLOTS -> PEER_GRPC_MAX_GAP_SLOTS (optional)
-# INGESTER_ROCKS_SYNC_INTERVAL_SECONDS and API_ROCKS_SYNC_INTERVAL_SECONDS -> ROCKS_SYNC_INTERVAL_SECONDS (optional)
-# INGESTER_GAPFILLER_PEER_ADDR -> GAPFILLER_PEER_ADDR (optional, only if run_gapfiller is set)
-# INGESTER_ACCOUNTS_BUFFER_SIZE -> ACCOUNT_PROCESSOR_BUFFER_SIZE (has default value, may be skipped)
-# INGESTER_ACCOUNTS_PARSING_WORKERS -> REDIS_ACCOUNTS_PARSING_WORKERS
-# INGESTER_ROCKS_FLUSH_BEFORE_BACKUP -> ROCKS_FLUSH_BEFORE_BACKUP
-# INGESTER_ROCKS_INTERVAL_IN_SECONDS -> ROCKS_INTERVAL_IN_SECONDS
-# INGESTER_TRANSACTIONS_PARSING_WORKERS -> REDIS_TRANSACTIONS_PARSING_WORKERS
-# INGESTER_SHOULD_REINGEST -> SHOULD_REINGEST
-# INGESTER_RUN_SEQUENCE_CONSISTENT_CHECKER -> RUN_SEQUENCE_CONSISTENT_CHECKER
-# INGESTER_RUN_PROFILING -> RUN_PROFILING
-# INGESTER_RUN_BUBBLEGUM_BACKFILLER -> RUN_BUBBLEGUM_BACKFILLER
-# Removed:
-# INGESTER_BACKFILL_RPC_ADDRESS, INGESTER_BACKFILLER_MODE, INGESTER_MESSAGE_SOURCE (redis is used as an only option as of now), INGESTER_DISABLE_SYNCHRONIZER (synchronizer is no longer part of the ingester), INGESTER_CHUNK_SIZE, INGESTER_PERMITTED_TASKS, INGESTER_TCP_CONFIG, INGESTER_WORKERS_COUNT, INGESTER_WAIT_PERIOD_SEC, INGESTER_SNAPSHOT_PARSING_WORKERS, INGESTER_SNAPSHOT_PARSING_BATCH_SIZE, INGESTER_SLOT_UNTIL, INGESTER_SLOT_START_FROM, INGESTER_RUN_FORK_CLEANER, INGESTER_RUN_DUMP_SYNCHRONIZE_ON_START

.github/workflows/cleanup.yml

@@ -0,0 +1,31 @@
+name: Cleanup Container Images
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    env:
+      REPO_NAME: ${GITHUB_REPOSITORY#*/}
+    strategy:
+      matrix:
+        binary: [ingester, das-api, slot_persister, backfill, synchronizer, rocksdb_backup]
+    steps:
+      - name: Set full package name
+        id: set_pkg
+        run: |
+          FULL_NAME="${REPO_NAME}-${{ matrix.binary }}"
+          echo "package_name=${FULL_NAME}" >> $GITHUB_OUTPUT
+          echo "Cleaning up package: ${FULL_NAME}"
+
+      - name: Cleanup package images
+        uses: ./.github/workflows/delete-container-versions.yml
+        with:
+          package_name: ${{ steps.set_pkg.outputs.package_name }}
+          keep_release: 30
+          keep_develop: 20
+          keep_other: 20
+          keep_untagged: 0

.github/workflows/delete-container-versions.yml

@@ -0,0 +1,66 @@
+name: Delete Container Versions
+
+on:
+  workflow_call:
+    inputs:
+      package_name:
+        required: true
+        type: string
+      keep_release:
+        required: false
+        type: number
+        default: 30
+      keep_develop:
+        required: false
+        type: number
+        default: 20
+      keep_other:
+        required: false
+        type: number
+        default: 20
+      keep_untagged:
+        required: false
+        type: number
+        default: 0
+
+jobs:
+  delete_old_packages:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Keep last release versions
+        uses: smartsquaregmbh/[email protected]
+        with:
+          organization: ${{ github.repository_owner }}
+          type: container
+          names: ${{ inputs.package_name }}
+          version-pattern: '^v\d+\.\d+\.\d+$'
+          keep: ${{ inputs.keep_release }}
+          dry-run: true
+
+      - name: Keep last develop versions
+        uses: smartsquaregmbh/[email protected]
+        with:
+          organization: ${{ github.repository_owner }}
+          type: container
+          names: ${{ inputs.package_name }}
+          version-pattern: '^develop-.*'
+          keep: ${{ inputs.keep_develop }}
+          dry-run: true
+
+      - name: Keep last other tagged versions
+        uses: smartsquaregmbh/[email protected]
+        with:
+          organization: ${{ github.repository_owner }}
+          type: container
+          names: ${{ inputs.package_name }}
+          version-pattern: '^(?!v\d+\.\d+\.\d+$)(?!latest)(?!develop-).*'
+          keep: ${{ inputs.keep_other }}
+          dry-run: true
+
+      - name: Keep last untagged versions
+        uses: actions/delete-package-versions@v5
+        with:
+          package-name: ${{ inputs.package_name }}
+          package-type: container
+          delete-only-untagged-versions: 'true'
+          min-versions-to-keep: ${{ inputs.keep_untagged }}

.github/workflows/docker.yml

@@ -8,13 +8,18 @@ on:
     branches: [develop]
     tags: ["v*"]
 
+# Add concurrency to cancel in-progress runs on the same ref
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 # Add permissions block for GitHub Container Registry access
 permissions:
   contents: read
   packages: write
 
 env:
-  PUSH_CONDITION: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && contains(fromJSON('["refs/head/main", "refs/head/develop"]'), github.event.workflow_dispatch.ref)) }}
+  PUSH_CONDITION: ${{ github.event_name == 'push' && (contains(fromJSON('["refs/heads/develop"]'), github.ref) || startsWith(github.ref, 'refs/tags/')) || github.event_name == 'workflow_dispatch' && contains(fromJSON('["refs/heads/develop"]'), github.ref) }}
 
 jobs:
   build-base-image:
@@ -23,7 +28,10 @@ jobs:
       version: ${{ steps.version.outputs.version }}
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -82,7 +90,10 @@ jobs:
             rocksdb_backup,
           ]
     steps:
-      - uses: actions/checkout@v4  # Need to checkout code for Dockerfile
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -142,7 +153,8 @@ jobs:
     runs-on: ubuntu-latest
     needs: [build-base-image, build-binary-images]
     steps:
-      - name: Repository dispatch
+      - name: Repository dispatch for development
+        if: startsWith(github.ref, 'refs/heads/develop')
         run: |
           curl -X POST \
             -H "Authorization: token ${{ secrets.DISPATCH_TOKEN_DEV }}" \
@@ -155,3 +167,18 @@ jobs:
                     "version": "${{ needs.build-base-image.outputs.version }}"
                   }
                 }'
+      
+      - name: Repository dispatch for production
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          curl -X POST \
+            -H "Authorization: token ${{ secrets.DISPATCH_TOKEN_PROD }}" \
+            -H "Accept: application/vnd.github+json" \
+            https://api.github.com/repos/adm-metaex/aura-config-prod/dispatches \
+            -d '{
+                  "event_type": "deploy",
+                  "client_payload": {
+                    "services": "${{ env.PUSH_CONDITION && 'ingester,slot_persister,backfill,api,synchronizer,rocksdb_backup' || '' }}",
+                    "version": "${{ needs.build-base-image.outputs.version }}"
+                  }
+                }'