diff --git a/.github/workflows/deploy-cf-latest-snapshot.yml b/.github/workflows/deploy-cf-latest-snapshot.yml deleted file mode 100644 index 93a0f605a..000000000 --- a/.github/workflows/deploy-cf-latest-snapshot.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Deploy CF latest-snapshot -on: - pull_request: - paths: - - 'cf/latest-snapshot/**' - - '.github/workflows/deploy-cf-latest-snapshot.yml' - push: - paths: - - 'cf/latest-snapshot/**' - - '.github/workflows/deploy-cf-latest-snapshot.yml' - workflow_dispatch: - -jobs: - check-and-deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Deployment check - uses: cloudflare/wrangler-action@v3 - with: - workingDirectory: "cf/latest-snapshot" - command: deploy --dry-run - - name: Deploy - if: github.ref == 'refs/heads/main' && ( github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) - uses: cloudflare/wrangler-action@v3 - with: - accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - workingDirectory: "cf/latest-snapshot" diff --git a/.github/workflows/deploy-cf-prune-latest.yml b/.github/workflows/deploy-cf-prune-latest.yml deleted file mode 100644 index ebd30a495..000000000 --- a/.github/workflows/deploy-cf-prune-latest.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Deploy CF prune-latest -on: - pull_request: - paths: - - 'cf/prune-latest/**' - - '.github/workflows/deploy-cf-prune-latest.yml' - push: - paths: - - 'cf/prune-latest/**' - - '.github/workflows/deploy-cf-prune-latest.yml' - workflow_dispatch: - -jobs: - check-and-deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Deployment check - uses: cloudflare/wrangler-action@v3 - with: - workingDirectory: "cf/prune-latest" - command: deploy --dry-run - - name: Deploy - if: github.ref == 'refs/heads/main' && ( github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) - uses: cloudflare/wrangler-action@v3 - with: - accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - workingDirectory: "cf/prune-latest" diff --git a/.github/workflows/deploy-daily-snapshot.yml b/.github/workflows/deploy-daily-snapshot.yml deleted file mode 100644 index 80979a5a5..000000000 --- a/.github/workflows/deploy-daily-snapshot.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Snapshot Service -concurrency: ci-${{ github.ref }}-snapshot-service - -on: - pull_request: - branches: - - main - paths: - - 'tf-managed/modules/daily-snapshot/**' - - 'tf-managed/scripts/**' - - 'tf-managed/live/environments/prod/applications/snapshot-service**' - # This needs to be declared explicitly so that the job is actually - # run when moved out of draft. - types: [opened, synchronize, reopened, ready_for_review] - push: - branches: - - main - paths: - - 'tf-managed/modules/daily-snapshot/**' - - 'tf-managed/scripts/**' - - 'tf-managed/live/environments/prod/applications/snapshot-service**' - workflow_dispatch: - -jobs: - deploy-daily-snapshot: - strategy: - matrix: - replica: ["snapshot-service", "snapshot-service-2"] - env: - TF_VAR_monitoring: "{ \"enable\": true,\"slack_enable\":true,\"slack_destination_id\":\"${{ secrets.SLACK_DESTINATION_ID }}\",\"slack_channel_id\":\"${{ secrets.SLACK_CHANNEL_ID }}\"}" - runs-on: ubuntu-latest - permissions: write-all - steps: - - name: Checkout the code - uses: actions/checkout@v4 - - # Using Custom Composite action in ./composite-action/terragrunt folder - - name: Composite Action for Deploying Terragrunt Resources - uses: ./composite-action/terragrunt - with: - do_token: ${{ secrets.DO_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - r2_access_key: ${{ secrets.R2_ACCESS_KEY }} - r2_secret_key: ${{ secrets.R2_SECRET_KEY }} - slack_token: ${{ secrets.SLACK_TOKEN }} - working_directory: tf-managed/live/environments/prod/applications/${{ matrix.replica }} - service_name: ${{ matrix.replica }} - new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} - new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }} - ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/snapshot-service-image.yml b/.github/workflows/snapshot-service-image.yml deleted file mode 100644 index 938ba5adb..000000000 --- a/.github/workflows/snapshot-service-image.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Snapshot Service Image - -# Cancel workflow if there is a new change to the branch. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -on: - push: - branches: [main] - merge_group: - pull_request: - branches: [main] - -jobs: - build-and-push-docker-image: - name: Build images and push to GHCR - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: List cached docker images - run: docker image ls - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Login to Github Packages - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # This step yields the following labels: - # - date+sha, e.g. 2023-01-19-da4692d, - # - latest, - - name: Docker Meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/chainsafe/forest-snapshot-service - tags: | - type=raw,value={{date 'YYYY-MM-DD'}}-{{sha}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build image and push to GitHub Container Registry - uses: docker/build-push-action@v6 - with: - context: ./images/snapshot-service/ - build-contexts: | - common=./tf-managed/scripts/ - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - push: ${{ github.ref == 'refs/heads/main' }} - - - name: List docker images - run: docker image ls diff --git a/README.md b/README.md index d33141c7c..adf32044f 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,7 @@ Feel free to contribute to the codebase by resolving any open issues, refactorin ## Questions Feel free to contact the team by creating an issue or raising a discussion [here](https://github.com/ChainSafe/forest/discussions) for more details on interacting with the infrastructure if the need arises during deployment. + +## Past Snapshot Service + +The snapshot service offered by the Forest team was transferred to the ChainSafe's infrastructure team for maintenance and further development. The previous implementation can be found [here](https://github.com/ChainSafe/forest-iac/pull/459). diff --git a/cf/forest-snapshot-listing/README.md b/cf/forest-snapshot-listing/README.md index ad0ad9397..59e5cf789 100644 --- a/cf/forest-snapshot-listing/README.md +++ b/cf/forest-snapshot-listing/README.md @@ -1,6 +1,6 @@ # Snapshot listing worker -This worker acts on endpoints at `https://forest-internal.chainsafe.dev/list**` and will list objects with `diff`, `lite`, and `latest` prefixes. +This worker acts on endpoints at `https://forest-internal.chainsafe.dev/list**` and will list objects with `diff` and `lite` prefixes. # Local deployment diff --git a/cf/forest-snapshot-listing/src/worker.ts b/cf/forest-snapshot-listing/src/worker.ts index 93cfd3a0b..9f29c7afb 100644 --- a/cf/forest-snapshot-listing/src/worker.ts +++ b/cf/forest-snapshot-listing/src/worker.ts @@ -59,10 +59,8 @@ export default { `; return new Response(html, { @@ -79,10 +77,6 @@ export default { return do_listing(env, 'mainnet/lite'); case '/list/calibnet/lite': return do_listing(env, 'calibnet/lite'); - case '/list/mainnet/latest': - return do_listing(env, 'mainnet/latest'); - case '/list/calibnet/latest': - return do_listing(env, 'calibnet/latest'); default: return new Response(`url: ${pathname}`); } diff --git a/cf/latest-snapshot/.editorconfig b/cf/latest-snapshot/.editorconfig deleted file mode 100644 index 64ab2601f..000000000 --- a/cf/latest-snapshot/.editorconfig +++ /dev/null @@ -1,13 +0,0 @@ -# http://editorconfig.org -root = true - -[*] -indent_style = tab -tab_width = 2 -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true - -[*.yml] -indent_style = space diff --git a/cf/latest-snapshot/.gitignore b/cf/latest-snapshot/.gitignore deleted file mode 100644 index e71378008..000000000 --- a/cf/latest-snapshot/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.wrangler diff --git a/cf/latest-snapshot/.prettierrc b/cf/latest-snapshot/.prettierrc deleted file mode 100644 index 557d4d5a7..000000000 --- a/cf/latest-snapshot/.prettierrc +++ /dev/null @@ -1,19 +0,0 @@ -{ - "semi": true, - "printWidth": 100, - "singleQuote": true, - "bracketSpacing": true, - "insertPragma": false, - "requirePragma": false, - "jsxSingleQuote": false, - "bracketSameLine": false, - "embeddedLanguageFormatting": "auto", - "htmlWhitespaceSensitivity": "css", - "vueIndentScriptAndStyle": true, - "quoteProps": "consistent", - "proseWrap": "preserve", - "trailingComma": "es5", - "arrowParens": "avoid", - "useTabs": true, - "tabWidth": 2 -} diff --git a/cf/latest-snapshot/README.md b/cf/latest-snapshot/README.md deleted file mode 100644 index bc36345d6..000000000 --- a/cf/latest-snapshot/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Snapshot redirect worker - -This worker acts on two endpoints: - -- `https://forest-internal.chainsafe.dev/latest/calibnet/` -- `https://forest-internal.chainsafe.dev/latest/mainnet/` - -- `https://forest-internal.chainsafe.dev/archive/calibnet/*` -- `https://forest-internal.chainsafe.dev/archive/mainnet/*` -- `https://forest-internal.chainsafe.dev/archive/historical/*` - -These links will download the latest available snapshot for calibnet and mainnet, respectively. - -# Local deployment - -First, login to Cloudflare with `wrangler login`. Then, use `wrangler dev --remote` to deploy a local version of this worker which will use the `forest-archive-dev` bucket rather than the production `forest-archive` bucket. Merging changes to this worker will automatically deploy them. diff --git a/cf/latest-snapshot/package-lock.json b/cf/latest-snapshot/package-lock.json deleted file mode 100644 index bc5993cca..000000000 --- a/cf/latest-snapshot/package-lock.json +++ /dev/null @@ -1,1210 +0,0 @@ -{ - "name": "forest-latest-snapshot", - "version": "0.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "forest-latest-snapshot", - "version": "0.0.0", - "devDependencies": { - "@cloudflare/workers-types": "^4.20230904.0", - "range-parser": "^1.2.1", - "wrangler": "^3.22.1" - } - }, - "node_modules/@cloudflare/kv-asset-handler": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.2.0.tgz", - "integrity": "sha512-MVbXLbTcAotOPUj0pAMhVtJ+3/kFkwJqc5qNOleOZTv6QkZZABDMS21dSrSlVswEHwrpWC03e4fWytjqKvuE2A==", - "dev": true, - "dependencies": { - "mime": "^3.0.0" - } - }, - "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20231030.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20231030.0.tgz", - "integrity": "sha512-J4PQ9utPxLya9yHdMMx3AZeC5M/6FxcoYw6jo9jbDDFTy+a4Gslqf4Im9We3aeOEdPXa3tgQHVQOSelJSZLhIw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20231030.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20231030.0.tgz", - "integrity": "sha512-WSJJjm11Del4hSneiNB7wTXGtBXI4QMCH9l5qf4iT5PAW8cESGcCmdHtWDWDtGAAGcvmLT04KNvmum92vRKKQQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20231030.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20231030.0.tgz", - "integrity": "sha512-2HUeRTvoCC17fxE0qdBeR7J9dO8j4A8ZbdcvY8pZxdk+zERU6+N03RTbk/dQMU488PwiDvcC3zZqS4gwLfVT8g==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20231030.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20231030.0.tgz", - "integrity": "sha512-4/GK5zHh+9JbUI6Z5xTCM0ZmpKKHk7vu9thmHjUxtz+o8Ne9DoD7DlDvXQWgMF6XGaTubDWyp3ttn+Qv8jDFuQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20231030.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20231030.0.tgz", - "integrity": "sha512-fb/Jgj8Yqy3PO1jLhk7mTrHMkR8jklpbQFud6rL/aMAn5d6MQbaSrYOCjzkKGp0Zng8D2LIzSl+Fc0C9Sggxjg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workers-types": { - "version": "4.20230914.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20230914.0.tgz", - "integrity": "sha512-OVeN4lFVu1O0PJGZ2d0FwpK8lelFcr33qYOgCh77ErEYmEBO4adwnIxcIsdQbFbhF0ffN6joiVcljD4zakdaeQ==", - "dev": true - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild-plugins/node-globals-polyfill": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@esbuild-plugins/node-globals-polyfill/-/node-globals-polyfill-0.2.3.tgz", - "integrity": "sha512-r3MIryXDeXDOZh7ih1l/yE9ZLORCd5e8vWg02azWRGj5SPTuoh69A2AIyn0Z31V/kHBfZ4HgWJ+OK3GTTwLmnw==", - "dev": true, - "peerDependencies": { - "esbuild": "*" - } - }, - "node_modules/@esbuild-plugins/node-modules-polyfill": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@esbuild-plugins/node-modules-polyfill/-/node-modules-polyfill-0.2.2.tgz", - "integrity": "sha512-LXV7QsWJxRuMYvKbiznh+U1ilIop3g2TeKRzUxOG5X3YITc8JyyTa90BmLwqqv0YnX4v32CSlG+vsziZp9dMvA==", - "dev": true, - "dependencies": { - "escape-string-regexp": "^4.0.0", - "rollup-plugin-node-polyfills": "^0.2.1" - }, - "peerDependencies": { - "esbuild": "*" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.19.tgz", - "integrity": "sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.19.tgz", - "integrity": "sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.19.tgz", - "integrity": "sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.19.tgz", - "integrity": "sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.19.tgz", - "integrity": "sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.19.tgz", - "integrity": "sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.19.tgz", - "integrity": "sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.19.tgz", - "integrity": "sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.19.tgz", - "integrity": "sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.19.tgz", - "integrity": "sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.19.tgz", - "integrity": "sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==", - "cpu": [ - "loong64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.19.tgz", - "integrity": "sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==", - "cpu": [ - "mips64el" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.19.tgz", - "integrity": "sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.19.tgz", - "integrity": "sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.19.tgz", - "integrity": "sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==", - "cpu": [ - "s390x" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.19.tgz", - "integrity": "sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.19.tgz", - "integrity": "sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.19.tgz", - "integrity": "sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.19.tgz", - "integrity": "sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.19.tgz", - "integrity": "sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.19.tgz", - "integrity": "sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.19.tgz", - "integrity": "sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@fastify/busboy": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", - "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", - "dev": true, - "engines": { - "node": ">=14" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", - "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "dev": true - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", - "dev": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.3.1", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.1.tgz", - "integrity": "sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==", - "dev": true, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/as-table": { - "version": "1.0.55", - "resolved": "https://registry.npmjs.org/as-table/-/as-table-1.0.55.tgz", - "integrity": "sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==", - "dev": true, - "dependencies": { - "printable-characters": "^1.0.42" - } - }, - "node_modules/binary-extensions": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/blake3-wasm": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", - "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", - "dev": true - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true - }, - "node_modules/capnp-ts": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/capnp-ts/-/capnp-ts-0.7.0.tgz", - "integrity": "sha512-XKxXAC3HVPv7r674zP0VC3RTXz+/JKhfyw94ljvF80yynK6VkTnqE3jMuN8b3dUVmmc43TjyxjW4KTsmB3c86g==", - "dev": true, - "dependencies": { - "debug": "^4.3.1", - "tslib": "^2.2.0" - } - }, - "node_modules/chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/data-uri-to-buffer": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-2.0.2.tgz", - "integrity": "sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==", - "dev": true - }, - "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/esbuild": { - "version": "0.17.19", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.19.tgz", - "integrity": "sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw==", - "dev": true, - "hasInstallScript": true, - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=12" - }, - "optionalDependencies": { - "@esbuild/android-arm": "0.17.19", - "@esbuild/android-arm64": "0.17.19", - "@esbuild/android-x64": "0.17.19", - "@esbuild/darwin-arm64": "0.17.19", - "@esbuild/darwin-x64": "0.17.19", - "@esbuild/freebsd-arm64": "0.17.19", - "@esbuild/freebsd-x64": "0.17.19", - "@esbuild/linux-arm": "0.17.19", - "@esbuild/linux-arm64": "0.17.19", - "@esbuild/linux-ia32": "0.17.19", - "@esbuild/linux-loong64": "0.17.19", - "@esbuild/linux-mips64el": "0.17.19", - "@esbuild/linux-ppc64": "0.17.19", - "@esbuild/linux-riscv64": "0.17.19", - "@esbuild/linux-s390x": "0.17.19", - "@esbuild/linux-x64": "0.17.19", - "@esbuild/netbsd-x64": "0.17.19", - "@esbuild/openbsd-x64": "0.17.19", - "@esbuild/sunos-x64": "0.17.19", - "@esbuild/win32-arm64": "0.17.19", - "@esbuild/win32-ia32": "0.17.19", - "@esbuild/win32-x64": "0.17.19" - } - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/estree-walker": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz", - "integrity": "sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==", - "dev": true - }, - "node_modules/exit-hook": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-2.2.1.tgz", - "integrity": "sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==", - "dev": true, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-source": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/get-source/-/get-source-2.0.12.tgz", - "integrity": "sha512-X5+4+iD+HoSeEED+uwrQ07BOQr0kEDFMVqqpBuI+RaZBpBpHCuXxo70bjar6f0b0u/DQJsJ7ssurpP0V60Az+w==", - "dev": true, - "dependencies": { - "data-uri-to-buffer": "^2.0.0", - "source-map": "^0.6.1" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", - "dev": true - }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/magic-string": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", - "integrity": "sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==", - "dev": true, - "dependencies": { - "sourcemap-codec": "^1.4.8" - } - }, - "node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "dev": true, - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/miniflare": { - "version": "3.20231030.4", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20231030.4.tgz", - "integrity": "sha512-7MBz0ArLuDop1WJGZC6tFgN6c5MRyDOIlxbm3yp0TRBpvDS/KsTuWCQcCjsxN4QQ5zvL3JTkuIZbQzRRw/j6ow==", - "dev": true, - "dependencies": { - "acorn": "^8.8.0", - "acorn-walk": "^8.2.0", - "capnp-ts": "^0.7.0", - "exit-hook": "^2.2.1", - "glob-to-regexp": "^0.4.1", - "source-map-support": "0.5.21", - "stoppable": "^1.1.0", - "undici": "^5.22.1", - "workerd": "1.20231030.0", - "ws": "^8.11.0", - "youch": "^3.2.2", - "zod": "^3.20.6" - }, - "bin": { - "miniflare": "bootstrap.js" - }, - "engines": { - "node": ">=16.13" - } - }, - "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/mustache": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", - "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", - "dev": true, - "bin": { - "mustache": "bin/mustache" - } - }, - "node_modules/nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/node-forge": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", - "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", - "dev": true, - "engines": { - "node": ">= 6.13.0" - } - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-to-regexp": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz", - "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==", - "dev": true - }, - "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/printable-characters": { - "version": "1.0.42", - "resolved": "https://registry.npmjs.org/printable-characters/-/printable-characters-1.0.42.tgz", - "integrity": "sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==", - "dev": true - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/resolve.exports": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", - "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/rollup-plugin-inject": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rollup-plugin-inject/-/rollup-plugin-inject-3.0.2.tgz", - "integrity": "sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w==", - "deprecated": "This package has been deprecated and is no longer maintained. Please use @rollup/plugin-inject.", - "dev": true, - "dependencies": { - "estree-walker": "^0.6.1", - "magic-string": "^0.25.3", - "rollup-pluginutils": "^2.8.1" - } - }, - "node_modules/rollup-plugin-node-polyfills": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/rollup-plugin-node-polyfills/-/rollup-plugin-node-polyfills-0.2.1.tgz", - "integrity": "sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA==", - "dev": true, - "dependencies": { - "rollup-plugin-inject": "^3.0.0" - } - }, - "node_modules/rollup-pluginutils": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz", - "integrity": "sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==", - "dev": true, - "dependencies": { - "estree-walker": "^0.6.1" - } - }, - "node_modules/selfsigned": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.1.1.tgz", - "integrity": "sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ==", - "dev": true, - "dependencies": { - "node-forge": "^1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", - "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", - "dev": true, - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/sourcemap-codec": { - "version": "1.4.8", - "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", - "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", - "deprecated": "Please use @jridgewell/sourcemap-codec instead", - "dev": true - }, - "node_modules/stacktracey": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/stacktracey/-/stacktracey-2.1.8.tgz", - "integrity": "sha512-Kpij9riA+UNg7TnphqjH7/CzctQ/owJGNbFkfEeve4Z4uxT5+JapVLFXcsurIfN34gnTWZNJ/f7NMG0E8JDzTw==", - "dev": true, - "dependencies": { - "as-table": "^1.0.36", - "get-source": "^2.0.12" - } - }, - "node_modules/stoppable": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", - "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==", - "dev": true, - "engines": { - "node": ">=4", - "npm": ">=6" - } - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", - "dev": true - }, - "node_modules/undici": { - "version": "5.28.4", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", - "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", - "dev": true, - "dependencies": { - "@fastify/busboy": "^2.0.0" - }, - "engines": { - "node": ">=14.0" - } - }, - "node_modules/workerd": { - "version": "1.20231030.0", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20231030.0.tgz", - "integrity": "sha512-+FSW+d31f8RrjHanFf/R9A+Z0csf3OtsvzdPmAKuwuZm/5HrBv83cvG9fFeTxl7/nI6irUUXIRF9xcj/NomQzQ==", - "dev": true, - "hasInstallScript": true, - "bin": { - "workerd": "bin/workerd" - }, - "engines": { - "node": ">=16" - }, - "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20231030.0", - "@cloudflare/workerd-darwin-arm64": "1.20231030.0", - "@cloudflare/workerd-linux-64": "1.20231030.0", - "@cloudflare/workerd-linux-arm64": "1.20231030.0", - "@cloudflare/workerd-windows-64": "1.20231030.0" - } - }, - "node_modules/wrangler": { - "version": "3.22.1", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.22.1.tgz", - "integrity": "sha512-fN7WOF6Ono/TV5V90PuJQNf0azS7B+5C/N/KRjqhlAIQBz+c0yLOGkF6kC/akxjr1yIAC9AzcPk9+OuTSq0C+g==", - "dev": true, - "dependencies": { - "@cloudflare/kv-asset-handler": "^0.2.0", - "@cspotcode/source-map-support": "0.8.1", - "@esbuild-plugins/node-globals-polyfill": "^0.2.3", - "@esbuild-plugins/node-modules-polyfill": "^0.2.2", - "blake3-wasm": "^2.1.5", - "chokidar": "^3.5.3", - "esbuild": "0.17.19", - "miniflare": "3.20231030.4", - "nanoid": "^3.3.3", - "path-to-regexp": "^6.2.0", - "resolve.exports": "^2.0.2", - "selfsigned": "^2.0.1", - "source-map": "0.6.1", - "xxhash-wasm": "^1.0.1" - }, - "bin": { - "wrangler": "bin/wrangler.js", - "wrangler2": "bin/wrangler.js" - }, - "engines": { - "node": ">=16.17.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/ws": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", - "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", - "dev": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xxhash-wasm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.0.2.tgz", - "integrity": "sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A==", - "dev": true - }, - "node_modules/youch": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/youch/-/youch-3.3.3.tgz", - "integrity": "sha512-qSFXUk3UZBLfggAW3dJKg0BMblG5biqSF8M34E06o5CSsZtH92u9Hqmj2RzGiHDi64fhe83+4tENFP2DB6t6ZA==", - "dev": true, - "dependencies": { - "cookie": "^0.5.0", - "mustache": "^4.2.0", - "stacktracey": "^2.1.8" - } - }, - "node_modules/zod": { - "version": "3.22.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", - "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", - "dev": true, - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/cf/latest-snapshot/package.json b/cf/latest-snapshot/package.json deleted file mode 100644 index 0b31d6fdd..000000000 --- a/cf/latest-snapshot/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "forest-latest-snapshot", - "version": "0.0.0", - "private": true, - "scripts": { - "deploy": "wrangler deploy src/index.ts", - "dev": "wrangler dev src/index.ts --local", - "start-stackblitz": "WRANGLER_SEND_METRICS=false wrangler dev src/index.ts --local" - }, - "devDependencies": { - "@cloudflare/workers-types": "^4.20230904.0", - "range-parser": "^1.2.1", - "wrangler": "^3.22.1" - } -} diff --git a/cf/latest-snapshot/src/index.ts b/cf/latest-snapshot/src/index.ts deleted file mode 100644 index d2ae51c10..000000000 --- a/cf/latest-snapshot/src/index.ts +++ /dev/null @@ -1,140 +0,0 @@ -import parseRange from 'range-parser'; -import { R2ObjectBody } from '@cloudflare/workers-types'; - -interface Env { - FOREST_ARCHIVE: R2Bucket; -} - -function basename(path: string) { - return path.split('/').reverse()[0]; -} - -enum SnapshotType { - latest = 'latest', - archive = 'archive', -} - -// Directly fetch the data for the latest snapshot of a given chain (eg. calibnet or mainnet) -async function get_latest( - req_headers: Headers, - env: Env, - path: string, - type: SnapshotType -): Promise { - let object: R2ObjectBody | R2Object | null = null; - - switch (type) { - case SnapshotType.latest: { - const listed = await env.FOREST_ARCHIVE.list({ prefix: path + '/latest/' }); - const latest = listed.objects.at(-1); - if (latest == undefined) { - return new Response(`No latest snapshot found ${path}`, { - status: 404, - }); - } - - // Response.redirect requires an absolute URL. Manually create the response to get around this. - return new Response('Found!', { - status: 302, - headers: { Location: '/archive/' + latest.key }, - }); - } - case SnapshotType.archive: { - object = await env.FOREST_ARCHIVE.get(path, { - range: req_headers, - onlyIf: req_headers, - }); - if (object === null) { - return new Response(`No archive snapshot found ${path}`, { - status: 404, - }); - } - break; - } - default: { - return new Response('Invalid Snapshot Type. Only archive OR latest is supported', { - status: 400, - }); - } - } - - const headers = new Headers(); - - // If the client requested a range, then we need to set the Content-Range header. - // The `parseRange` returns an error code if the range is not satisfiable (or not specified), - // so we can handle the range request only if it returns an array. - let status = 200; - const range = parseRange(object.size, req_headers.get('range') || ''); - if (Array.isArray(range)) { - // R2Object doesn't support multiple ranges, so we only use the first one. - // Throw an error if there are more than one range. - if (range.length > 1) { - return new Response('Multiple ranges are not supported', { - status: 416, // Range Not Satisfiable - }); - } - - const r = range[0]; - headers.set('Content-Range', `${r.type} ${r.start}-${r.end}/${object.size}`); - headers.set('Content-Length', `${r.end - r.start + 1}`); - status = 206; // Partial Content - } else { - headers.set('Content-Length', object.size.toString()); - } - - object.writeHttpMetadata(headers); - headers.set('etag', object.httpEtag); - headers.set('Accept-Ranges', 'bytes'); - const encoded_name = encodeURIComponent(basename(object.key)); - // Tell browsers and aria2c which filename to use. For 'wget', you have to use `--trust-server-names`. - headers.set( - 'Content-Disposition', - `attachment; filename*=UTF-8''${encoded_name}; filename="${encoded_name}"` - ); - - if ('body' in object) { - return new Response(object.body, { - headers, - status, - }); - } else { - return new Response(null, { - headers, - status, - }); - } -} - -export default { - async fetch(request: Request, env: Env): Promise { - // Disallow any other request method except HEAD and GET, they are not sensible in the context - // of fetching a snapshot. - switch (request.method) { - case 'HEAD': - case 'GET': { - const url = new URL(request.url); - const path = url.pathname.match(/\/archive\/(\S*)/); - if (path != null && path.length > 1) { - return await get_latest(request.headers, env, path[1], SnapshotType.archive); - } - - const chain = url.pathname.match(/\/latest\/(\w*)/); - if (chain != null && chain.length > 1) { - return await get_latest(request.headers, env, chain[1], SnapshotType.latest); - } - - return new Response('path not found', { - status: 404, - }); - } - default: { - return new Response('Method not allowed', { - status: 405, - headers: { - Allow: 'GET, HEAD', - }, - }); - } - } - }, -}; diff --git a/cf/latest-snapshot/tsconfig.json b/cf/latest-snapshot/tsconfig.json deleted file mode 100644 index 3ca1f9899..000000000 --- a/cf/latest-snapshot/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "noEmit": true, - "module": "esnext", - "target": "esnext", - "lib": ["esnext"], - "strict": true, - "moduleResolution": "node", - "types": ["@cloudflare/workers-types", "@types/jest"] - }, - "exclude": ["node_modules", "dist"] -} diff --git a/cf/latest-snapshot/wrangler.toml b/cf/latest-snapshot/wrangler.toml deleted file mode 100644 index 1d0f90bc0..000000000 --- a/cf/latest-snapshot/wrangler.toml +++ /dev/null @@ -1,22 +0,0 @@ -name = "forest-latest-snapshot" -main = "./src/index.ts" - -compatibility_date = "2022-06-30" -logpush = true - -routes = [ - { pattern = "forest-internal.chainsafe.dev/latest/calibnet", zone_name = "chainsafe.dev" }, - { pattern = "forest-internal.chainsafe.dev/latest/mainnet", zone_name = "chainsafe.dev" }, - { pattern = "forest-internal.chainsafe.dev/latest/calibnet/", zone_name = "chainsafe.dev" }, - { pattern = "forest-internal.chainsafe.dev/latest/mainnet/", zone_name = "chainsafe.dev" }, - -# below endpoints are for the archive - { pattern = "forest-internal.chainsafe.dev/archive/calibnet/*", zone_name = "chainsafe.dev" }, - { pattern = "forest-internal.chainsafe.dev/archive/mainnet/*", zone_name = "chainsafe.dev" }, - { pattern = "forest-internal.chainsafe.dev/archive/historical/*", zone_name = "chainsafe.dev" } -] - -[[r2_buckets]] -binding = 'FOREST_ARCHIVE' # can be any valid JavaScript variable name -bucket_name = 'forest-archive' -preview_bucket_name = 'forest-archive-dev' diff --git a/cf/prune-latest/README.md b/cf/prune-latest/README.md deleted file mode 100644 index e9da71dbe..000000000 --- a/cf/prune-latest/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Deployment - -This worker is automatically deployed when modified. To test locally, run `wrangler dev`. This will run the worker against the development bucket `forest-archive-dev`. Once the worker is deployed to production, it'll use the `forest-archive` bucket. - -# Pruning - -We upload new Filecoin snapshots to CloudFlare every hour and keep only the 10 most recent. The CloudFlare worker script is triggered automatically every hour. diff --git a/cf/prune-latest/package.json b/cf/prune-latest/package.json deleted file mode 100644 index 218df7a32..000000000 --- a/cf/prune-latest/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "template-worker-r2", - "version": "0.0.0", - "private": true, - "scripts": { - "deploy": "wrangler deploy src/index.ts", - "dev": "wrangler dev src/index.ts --local", - "start-stackblitz": "WRANGLER_SEND_METRICS=false wrangler dev src/index.ts --local" - }, - "devDependencies": { - "@cloudflare/workers-types": "^4.20230904.0", - "wrangler": "^3.0.0" - } -} diff --git a/cf/prune-latest/src/index.ts b/cf/prune-latest/src/index.ts deleted file mode 100644 index 492102d2d..000000000 --- a/cf/prune-latest/src/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -export interface Env { - FOREST_ARCHIVE: R2Bucket; -} - -// Number of recent snapshots to keep. Roughly 1 new snapshot is uploaded every hour. -// Keep 24*14=336 snapshots for approximately 14 days of retention. -const KEEP_COUNT: number = 24 * 14; - -async function prune(env: Env, chain: string): Promise { - const listed = await env.FOREST_ARCHIVE.list({ prefix: chain + "/latest/" }); - // objects are listed chronologically. Reverse to keep the newest snapshots. - listed.objects.reverse(); - for (let i: number = KEEP_COUNT; i < listed.objects.length; i++) { - await env.FOREST_ARCHIVE.delete(listed.objects[i].key); - } - const pruned = Math.max(listed.objects.length - KEEP_COUNT, 0); - const kept = listed.objects.length - pruned; - return `Pruned: ${pruned}. Kept: ${kept}`; -} - -export default { - async scheduled(_event: Event, env: Env) { - await prune(env, "calibnet"); - await prune(env, "mainnet"); - }, - async fetch(request: Request, env: Env): Promise { - const calibnet = await prune(env, "calibnet"); - const mainnet = await prune(env, "mainnet"); - return new Response(`Calibnet: ${calibnet}\nMainnet: ${mainnet}\n`); - }, -}; diff --git a/cf/prune-latest/tsconfig.json b/cf/prune-latest/tsconfig.json deleted file mode 100644 index 3ca1f9899..000000000 --- a/cf/prune-latest/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "noEmit": true, - "module": "esnext", - "target": "esnext", - "lib": ["esnext"], - "strict": true, - "moduleResolution": "node", - "types": ["@cloudflare/workers-types", "@types/jest"] - }, - "exclude": ["node_modules", "dist"] -} diff --git a/cf/prune-latest/wrangler.toml b/cf/prune-latest/wrangler.toml deleted file mode 100644 index 705d4194e..000000000 --- a/cf/prune-latest/wrangler.toml +++ /dev/null @@ -1,16 +0,0 @@ -name = "forest-prune-latest" -main = "./src/index.ts" - -compatibility_date = "2022-06-30" - -routes = [ - { pattern = "forest-internal.chainsafe.dev/prune/", zone_name = "chainsafe.dev" }, -] - -[[r2_buckets]] -binding = 'FOREST_ARCHIVE' # can be any valid JavaScript variable name -bucket_name = 'forest-archive' -preview_bucket_name = 'forest-archive-dev' - -[triggers] -crons = ["0 * * * *"] diff --git a/composite-action/terragrunt/action.yml b/composite-action/terragrunt/action.yml index d0ee8beb3..be4052458 100644 --- a/composite-action/terragrunt/action.yml +++ b/composite-action/terragrunt/action.yml @@ -11,10 +11,10 @@ inputs: description: 'The DigitalOcean access token to use for deploying the infrastructure' required: true aws_access_key_id: - description: 'S3 access keys id used by terraform and service like sync check, Deploy Snapshot Service etc' + description: 'S3 access keys id used by terraform and service like sync check' required: true aws_secret_access_key: - description: 'S3 secret access keys used by terraform and service like sync check, Deploy Snapshot Service etc' + description: 'S3 secret access keys used by terraform and service like sync check' required: true working_directory: description: 'The working Directory' diff --git a/images/snapshot-service/.dockerignore b/images/snapshot-service/.dockerignore deleted file mode 100644 index dd449725e..000000000 --- a/images/snapshot-service/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -*.md diff --git a/images/snapshot-service/Dockerfile b/images/snapshot-service/Dockerfile deleted file mode 100644 index 1a23beb53..000000000 --- a/images/snapshot-service/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Snapshot service Dockerfile. -# It is meant to produce a single snapshot of the given chain in the Filecoin network and upload it to S3 (preferably Cloudflare R2, it should work for other providers as well, but it wasn't tested). -FROM docker:24 -LABEL org.opencontainers.image.description "Forest snapshot service generator and uploader for Filecoin" - -RUN apk add --no-cache \ - ruby \ - ruby-dev \ - docker \ - bash && \ - gem install \ - docker-api \ - slack-ruby-client \ - activesupport - -COPY ./src /opt/snapshot-service - -# `common` is defined via the `--build-context` flag in the `docker build` command, e.g., -# `docker build --build-context common=../../tf-managed/scripts/ -t ghcr.io/chainsafe/forest-snapshot-service:latest .` -# TODO: Change this once `sync-check` is fully-dockerized as well. -# hadolint ignore=DL3022 -COPY --from=common ruby_common /opt/snapshot-service/ruby_common - -WORKDIR /opt/snapshot-service - -CMD ["bash", "run.sh"] diff --git a/images/snapshot-service/README.md b/images/snapshot-service/README.md deleted file mode 100644 index 050e1dd62..000000000 --- a/images/snapshot-service/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Forest snapshot service - -This service serves as a Filecoin snapshot generator and uploader. Supported networks are [calibnet](https://docs.filecoin.io/networks/calibration) and [mainnet](https://docs.filecoin.io/networks/mainnet). All S3-compatible providers should work correctly, though it was tested exclusively on Cloudflare R2. - -## Building the image - -```bash -docker build --build-context common=../../tf-managed/scripts/ -t : . -``` - -## Running the Forest snapshot service - -The container needs additional privileges and access to the docker socket to issue other `docker` commands. - -This command will generate a snapshot for the given network and upload it to an S3 bucket. -```bash -docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env-file --env NETWORK_CHAIN= ghcr.io/chainsafe/forest-snapshot-service:edge -``` - -## Variables (all required) - -```bash -# Details for the snapshot upload -R2_ACCESS_KEY= -R2_SECRET_KEY= -R2_ENDPOINT= -SNAPSHOT_BUCKET= - -# Details for the Slack notifications -SLACK_API_TOKEN= -SLACK_NOTIFICATION_CHANNEL= - -# Network chain - can be either `mainnet` or `calibnet` -NETWORK_CHAIN= -# Forest tag to use. `latest` is the newest stable version. -# See [Forest packages](https://github.com/ChainSafe/forest/pkgs/container/forest) for more. -FOREST_TAG= -``` diff --git a/images/snapshot-service/src/daily_snapshot.rb b/images/snapshot-service/src/daily_snapshot.rb deleted file mode 100644 index 8d3f2d213..000000000 --- a/images/snapshot-service/src/daily_snapshot.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -require_relative 'ruby_common/slack_client' -require_relative 'ruby_common/docker_utils' -require_relative 'ruby_common/utils' - -require 'date' -require 'logger' -require 'fileutils' - -SLACK_TOKEN = get_and_assert_env_variable 'SLACK_API_TOKEN' -CHANNEL = get_and_assert_env_variable 'SLACK_NOTIFICATION_CHANNEL' - -CHAIN_NAME = ARGV[0] -raise 'No chain name supplied. Please provide chain identifier, e.g. calibnet or mainnet' if ARGV.empty? - -# Current datetime, to append to the log files -DATE = Time.new.strftime '%FT%H:%M:%S' - -FileUtils.mkdir_p 'logs' -LOG_EXPORT_SCRIPT_RUN = "logs/#{CHAIN_NAME}_#{DATE}_script_run.txt" -LOG_EXPORT_DAEMON = "logs/#{CHAIN_NAME}_#{DATE}_daemon.txt" -LOG_EXPORT_METRICS = "logs/#{CHAIN_NAME}_#{DATE}_metrics.txt" - -client = SlackClient.new CHANNEL, SLACK_TOKEN -logger = Logger.new($stdout) - -# conditionally add timestamps to logs without timestamps -add_timestamps_cmd = <<~CMD - awk '{ - if ($0 !~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z/) - print strftime("[%Y-%m-%d %H:%M:%S]"), $0; - else - print $0; - fflush(); - }' -CMD - -upload_cmd = <<~CMD.chomp - set -o pipefail && \ - timeout -s SIGKILL 8h ./upload_snapshot.sh #{CHAIN_NAME} #{LOG_EXPORT_DAEMON} #{LOG_EXPORT_METRICS} | \ - #{add_timestamps_cmd} -CMD - -# The command needs to be run indirectly to avoid syntax errors in the shell. -logger.info "Running snapshot export script for #{CHAIN_NAME}..." -snapshot_uploaded = system('bash', '-c', upload_cmd, %i[out err] => LOG_EXPORT_SCRIPT_RUN) -logger.info "Snapshot export script finished for #{CHAIN_NAME}." - -if snapshot_uploaded - # This log message is important, as it is used by the monitoring tools to determine whether the snapshot was - # successfully uploaded. - logger.info "Snapshot uploaded for #{CHAIN_NAME}." -else - logger.error "Snapshot upload failed for #{CHAIN_NAME}." - client.post_message "⛔ Snapshot failed for #{CHAIN_NAME}. 🔥🌲🔥 " - # attach the log file and print the contents to STDOUT - [LOG_EXPORT_SCRIPT_RUN, LOG_EXPORT_DAEMON, LOG_EXPORT_METRICS].each do |log_file| - client.attach_files(log_file) if File.exist?(log_file) - end -end - -[LOG_EXPORT_SCRIPT_RUN, LOG_EXPORT_DAEMON, LOG_EXPORT_METRICS].each do |log_file| - logger.info "Snapshot export log:\n#{File.read(log_file)}\n\n" if File.exist?(log_file) -end diff --git a/images/snapshot-service/src/run.sh b/images/snapshot-service/src/run.sh deleted file mode 100644 index 7dd9577b6..000000000 --- a/images/snapshot-service/src/run.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Assert that all required environment variables are set -: "${R2_ACCESS_KEY:?}" -: "${R2_SECRET_KEY:?}" -: "${R2_ENDPOINT:?}" -: "${SNAPSHOT_BUCKET:?}" -: "${SLACK_API_TOKEN:?}" -: "${SLACK_NOTIFICATION_CHANNEL:?}" -: "${NETWORK_CHAIN:?}" -: "${FOREST_TAG:?}" - -ruby daily_snapshot.rb "$NETWORK_CHAIN" diff --git a/images/snapshot-service/src/upload_snapshot.sh b/images/snapshot-service/src/upload_snapshot.sh deleted file mode 100755 index d0c39aeab..000000000 --- a/images/snapshot-service/src/upload_snapshot.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash - -# If Forest hasn't synced to the network after 6 hours, something has gone wrong. -SYNC_TIMEOUT=6h - -if [[ $# != 3 ]]; then - echo "Usage: bash $0 CHAIN_NAME LOG_EXPORT_DAEMON LOG_EXPORT_METRICS" - exit 1 -fi - -CHAIN_NAME=$1 -LOG_EXPORT_DAEMON=$2 -LOG_EXPORT_METRICS=$3 - -# Make sure we have the most recent Forest image -docker pull ghcr.io/chainsafe/forest:"${FOREST_TAG}" - -# Sync and export is done in a single container to make sure everything is -# properly cleaned up. -COMMANDS=$(cat << HEREDOC -set -eux - -# Install utility binaries that do not come with the image. -# This assumes the container was started as a superuser. -apt-get update && apt-get install -y curl aria2 - -# Switch back to the service user for other service commands. -su - forest - -function add_timestamps { - while IFS= read -r line; do - echo "\$(date +'%Y-%m-%d %H:%M:%S') \$line" - done -} - -# periodically write metrics to a file -# this is done in a separate process to avoid blocking the sync process -# and to ensure that the metrics are written even if it crashes -function write_metrics { - while true; do - { - curl --silent --fail --max-time 5 --retry 5 --retry-delay 2 --retry-max-time 10 http://localhost:6116/metrics || true - } | add_timestamps >> "$LOG_EXPORT_METRICS" - sleep 15 - done -} - -function print_forest_logs { - cat forest.err forest.out > $LOG_EXPORT_DAEMON -} -trap print_forest_logs EXIT - -echo "[client]" > config.toml -echo 'data_dir = "/home/forest/forest_db"' >> config.toml -echo 'encrypt_keystore = false' >> config.toml - -echo "Chain: $CHAIN_NAME" - -# spawn a task in the background to periodically write Prometheus metrics to a file -( - set +x # Disable debugging for this subshell to keep the logs clean. - write_metrics -) & - -forest-tool db destroy --force --config config.toml --chain "$CHAIN_NAME" - -# Workaround for https://github.com/ChainSafe/forest/issues/3715 -# Normally, Forest should automatically download the latest snapshot. However, the performance -# of the download gets randomly bad, and the download times out. -# Retry logic, because CF occassionally returns 500 (not 503) errors. -for i in {1..5}; do aria2c -x5 https://forest-internal.chainsafe.dev/latest/$CHAIN_NAME/ && break || sleep 15; done - -forest --config config.toml --chain "$CHAIN_NAME" --consume-snapshot *.car.zst --halt-after-import - -forest --config config.toml --chain "$CHAIN_NAME" --no-gc --save-token=token.txt --target-peer-count 500 --detach -timeout "$SYNC_TIMEOUT" forest-cli sync wait -forest-cli snapshot export -o forest_db/ -forest-cli --token=\$(cat token.txt) shutdown --force - -# Snapshot is exported, remove the Forest DB to limit space usage. -forest-tool db destroy --force --config config.toml --chain "$CHAIN_NAME" - -# Run full checks only for calibnet, given that it takes too long for mainnet. -if [ "$CHAIN_NAME" = "calibnet" ]; then - timeout 30m forest-tool snapshot validate --check-network "$CHAIN_NAME" forest_db/forest_snapshot_*.forest.car.zst -else - forest-tool archive info forest_db/forest_snapshot_*.forest.car.zst - timeout 30m forest-tool snapshot validate --check-links 0 --check-network "$CHAIN_NAME" --check-stateroots 5 forest_db/forest_snapshot_*.forest.car.zst -fi - -# Kill the metrics writer process -kill %1 - -HEREDOC -) - -# Stop any lingering docker containers -CONTAINER_NAME="forest-snapshot-upload-node-$CHAIN_NAME" -docker stop "$CONTAINER_NAME" || true -docker rm --force "$CONTAINER_NAME" - -# Cleanup volumes from the previous if any. -DB_VOLUME="${CHAIN_NAME}_db" -LOG_VOLUME="${CHAIN_NAME}_logs" -docker volume rm "${DB_VOLUME}" || true -docker volume rm "${LOG_VOLUME}" || true - -# Run forest and generate a snapshot in the `DB_VOLUME` volume. -docker run \ - --name "$CONTAINER_NAME" \ - --user root \ - -v "${DB_VOLUME}:/home/forest/forest_db" \ - -v "${LOG_VOLUME}:/home/forest/logs" \ - --entrypoint /bin/bash \ - ghcr.io/chainsafe/forest:"${FOREST_TAG}" \ - -c "$COMMANDS" - -generation_result=$? - -# Copy the logs to the current container. Mounting won't work because it would use the real host's filesystem. -docker cp "$CONTAINER_NAME":/home/forest/logs/. "$(dirname "$LOG_EXPORT_DAEMON")" -docker rm --force "$CONTAINER_NAME" - -if [[ $generation_result != 0 ]]; then - echo "Snapshot generation failed" - exit 1 -fi - -# Mount the snapshot volume and copy the snapshot to the S3 bucket. -docker run -v "${DB_VOLUME}":/opt/snapshots --rm --entrypoint /bin/bash --env AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY" --env AWS_SECRET_ACCESS_KEY="$R2_SECRET_KEY" \ - public.ecr.aws/aws-cli/aws-cli:2.15.18 \ - -c "aws configure set default.s3.multipart_chunksize 4GB && aws --endpoint ${R2_ENDPOINT} s3 cp --no-progress /opt/snapshots/forest_snapshot_${CHAIN_NAME}*.forest.car.zst s3://${SNAPSHOT_BUCKET}/${CHAIN_NAME}/latest/" || exit 1 - -docker volume rm "${DB_VOLUME}" || true diff --git a/tf-managed/live/README.md b/tf-managed/live/README.md index 0c059def1..374630022 100644 --- a/tf-managed/live/README.md +++ b/tf-managed/live/README.md @@ -46,10 +46,6 @@ Each environment contains its respective `applications/`. A `base-infrastructure ``` └── applications - ├── snapshot-monitoring - │   └── terragrunt.hcl - ├── snapshot-service - │   └── terragrunt.hcl └── sync-check └── terragrunt.hcl ``` diff --git a/tf-managed/live/environments/dev/applications/snapshot-service/terragrunt.hcl b/tf-managed/live/environments/dev/applications/snapshot-service/terragrunt.hcl deleted file mode 100644 index ea35a0d04..000000000 --- a/tf-managed/live/environments/dev/applications/snapshot-service/terragrunt.hcl +++ /dev/null @@ -1,22 +0,0 @@ -# Automatically find the root terragrunt.hcl and inherit its -# configuration -include "root" { - path = find_in_parent_folders() -} - -# Load the actual Terraform module -terraform { - source = format("%s/../modules/daily-snapshot", get_parent_terragrunt_dir()) -} - -inputs = { - name = "forest-snapshot" - size = "s-4vcpu-16gb-amd" - r2_endpoint = "https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com/" - forest_tag = "latest-fat" - snapshot_bucket = "forest-archive-dev" - - monitoring = { - enable = true, - } -} diff --git a/tf-managed/live/environments/prod/applications/snapshot-service-2/terragrunt.hcl b/tf-managed/live/environments/prod/applications/snapshot-service-2/terragrunt.hcl deleted file mode 100644 index 5c3dd37c9..000000000 --- a/tf-managed/live/environments/prod/applications/snapshot-service-2/terragrunt.hcl +++ /dev/null @@ -1,23 +0,0 @@ -# Automatically find the root terragrunt.hcl and inherit its -# configuration -include "root" { - path = find_in_parent_folders() -} - -# Load the actual Terraform module -terraform { - source = format("%s/../modules/daily-snapshot", get_parent_terragrunt_dir()) -} - -inputs = { - name = "forest-snapshot-2" - size = "s-4vcpu-16gb-320gb-intel" - r2_endpoint = "https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com/" - forest_tag = "v0.19.0-fat" - snapshot_bucket = "forest-archive" - region = "sfo3" - - monitoring = { - enable = true, - } -} diff --git a/tf-managed/live/environments/prod/applications/snapshot-service/terragrunt.hcl b/tf-managed/live/environments/prod/applications/snapshot-service/terragrunt.hcl deleted file mode 100644 index c66f4d683..000000000 --- a/tf-managed/live/environments/prod/applications/snapshot-service/terragrunt.hcl +++ /dev/null @@ -1,22 +0,0 @@ -# Automatically find the root terragrunt.hcl and inherit its -# configuration -include "root" { - path = find_in_parent_folders() -} - -# Load the actual Terraform module -terraform { - source = format("%s/../modules/daily-snapshot", get_parent_terragrunt_dir()) -} - -inputs = { - name = "forest-snapshot" - size = "s-4vcpu-16gb-amd" - r2_endpoint = "https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com/" - forest_tag = "v0.19.0-fat" - snapshot_bucket = "forest-archive" - - monitoring = { - enable = true, - } -} diff --git a/tf-managed/modules/daily-snapshot/firewall.tf b/tf-managed/modules/daily-snapshot/firewall.tf deleted file mode 100644 index 0a2ee0227..000000000 --- a/tf-managed/modules/daily-snapshot/firewall.tf +++ /dev/null @@ -1,48 +0,0 @@ -resource "digitalocean_firewall" "forest-firewall" { - name = format("%s-%s", var.environment, var.name) - - inbound_rule { - protocol = "tcp" - port_range = "22" - source_addresses = var.source_addresses - } - - inbound_rule { - protocol = "tcp" - port_range = "2345" - source_addresses = var.source_addresses - } - - inbound_rule { - protocol = "tcp" - port_range = "80" - source_addresses = var.source_addresses - } - - inbound_rule { - protocol = "udp" - port_range = "53" - source_addresses = var.source_addresses - } - - outbound_rule { - protocol = "tcp" - port_range = "all" - destination_addresses = var.destination_addresses - } - - outbound_rule { - protocol = "udp" - port_range = "53" - destination_addresses = var.destination_addresses - } - - # NTP - outbound_rule { - protocol = "udp" - port_range = "123" - destination_addresses = var.destination_addresses - } - - droplet_ids = [digitalocean_droplet.forest.id] -} diff --git a/tf-managed/modules/daily-snapshot/main.tf b/tf-managed/modules/daily-snapshot/main.tf deleted file mode 100644 index e657b151c..000000000 --- a/tf-managed/modules/daily-snapshot/main.tf +++ /dev/null @@ -1,115 +0,0 @@ -# This terraform script executes the following steps: -# - Zip the ruby and shell script files (the hash of this zip file is used to -# determine when to re-deploy the service) -# - Boot a new droplet -# - Copy over the zip file -# - Run the init.sh script in the background - -// Ugly hack because 'archive_file' cannot mix files and folders. -data "external" "sources_tar" { - program = ["bash", "${path.module}/prep_sources.sh", path.module] -} - - -data "local_file" "sources" { - filename = data.external.sources_tar.result.path -} - -// Note: The init.sh file is also included in the sources.zip such that the hash -// of the archive captures the entire state of the machine. -// This is a workaround, and because of this, we need to suppress the tflint warning here -// for unused declarations related to the 'init.sh' file. tflint-ignore: terraform_unused_declarations -data "local_file" "init" { - filename = "${path.module}/service/init.sh" -} - -data "digitalocean_ssh_keys" "keys" { - sort { - key = "name" - direction = "asc" - } -} - -# Required environment variables for the snapshot service itself. -locals { - env_content = <<-EOT - R2_ACCESS_KEY=${var.R2_ACCESS_KEY} - R2_SECRET_KEY=${var.R2_SECRET_KEY} - R2_ENDPOINT=${var.r2_endpoint} - SNAPSHOT_BUCKET=${var.snapshot_bucket} - SLACK_API_TOKEN=${var.slack_token} - SLACK_NOTIFICATION_CHANNEL=${var.slack_channel} - FOREST_TAG=${var.forest_tag} - EOT -} - -locals { - init_commands = ["cd /root/", - "tar xf sources.tar", - "echo '${local.env_content}' >> /root/.forest_env", - <<-EOT - export NEW_RELIC_API_KEY=${var.new_relic_api_key} - export NEW_RELIC_ACCOUNT_ID=${var.new_relic_account_id} - export NEW_RELIC_REGION=${var.new_relic_region} - nohup sh ./init.sh > init_log.txt & - EOT - , - # Exiting without a sleep sometimes kills the script :-/ - "sleep 60s" - ] - - service_name = format("%s-%s", var.environment, var.name) -} - -resource "digitalocean_droplet" "forest" { - image = var.image - name = local.service_name - region = var.region - size = var.size - # Re-initialize resource if this hash changes: - user_data = join("-", [data.local_file.sources.content_sha256, sha256(join("", local.init_commands))]) - tags = ["iac", var.environment] - ssh_keys = data.digitalocean_ssh_keys.keys.ssh_keys[*].fingerprint - monitoring = true - - graceful_shutdown = false - - connection { - host = self.ipv4_address - user = "root" - type = "ssh" - timeout = "30m" - } - - # Push the sources.tar file to the newly booted droplet - provisioner "file" { - source = data.local_file.sources.filename - destination = "/root/sources.tar" - } - - provisioner "remote-exec" { - inline = local.init_commands - } -} - - -data "digitalocean_project" "forest_project" { - name = var.project -} - -# Connect the droplet to the forest project (otherwise it ends up in -# "ChainBridge" which is the default project) -resource "digitalocean_project_resources" "connect_forest_project" { - project = data.digitalocean_project.forest_project.id - resources = [digitalocean_droplet.forest.urn] -} - -module "monitoring" { - count = var.monitoring.enable ? 1 : 0 - source = "./monitoring" - service_name = local.service_name - alert_email = var.monitoring.alert_email - slack_enable = var.monitoring.slack_enable - slack_destination_id = var.monitoring.slack_destination_id - slack_channel_id = var.monitoring.slack_channel_id -} diff --git a/tf-managed/modules/daily-snapshot/monitoring/main.tf b/tf-managed/modules/daily-snapshot/monitoring/main.tf deleted file mode 100644 index 19079d5ca..000000000 --- a/tf-managed/modules/daily-snapshot/monitoring/main.tf +++ /dev/null @@ -1,120 +0,0 @@ -resource "newrelic_alert_policy" "alert" { - name = format("%s alert policy", var.service_name) -} - -locals { - enable_email = var.alert_email != "" -} - -resource "newrelic_nrql_alert_condition" "disk_space" { - policy_id = newrelic_alert_policy.alert.id - type = "static" - name = "High Disk Utilization" - description = "Alert when disk space usage is high on an the service host" - enabled = true - - nrql { - query = "SELECT latest(diskUsedPercent) FROM StorageSample where entityName = '${var.service_name}'" - } - - critical { - operator = "above" - threshold = 95.0 - threshold_duration = 300 - threshold_occurrences = "ALL" - } - - warning { - operator = "above" - threshold = 85.0 - threshold_duration = 300 - threshold_occurrences = "ALL" - } -} - -resource "newrelic_notification_destination" "email" { - count = local.enable_email ? 1 : 0 - name = format("%s email", var.service_name) - type = "EMAIL" - - property { - key = "email" - value = var.alert_email - } -} - -resource "newrelic_notification_channel" "email-channel" { - count = local.enable_email ? 1 : 0 - name = format("%s email", var.service_name) - type = "EMAIL" - product = "IINT" - destination_id = newrelic_notification_destination.email[0].id - - property { - key = "subject" - value = format("%s alert", var.service_name) - } -} - -resource "newrelic_notification_channel" "slack-channel" { - count = var.slack_enable ? 1 : 0 - name = format("%s slack", var.service_name) - type = "SLACK" - destination_id = var.slack_destination_id - product = "IINT" - - property { - key = "channelId" - value = var.slack_channel_id - } - - property { - key = "customDetailsSlack" - value = "issue id - {{issueId}}" - } -} - - -resource "newrelic_workflow" "alerting-workflow-mails" { - count = local.enable_email ? 1 : 0 - name = format("%s mail alerting workflow", var.service_name) - muting_rules_handling = "NOTIFY_ALL_ISSUES" - - issues_filter { - name = format("%s alerting workflow filter", var.service_name) - type = "FILTER" - - predicate { - attribute = "labels.policyIds" - operator = "EXACTLY_MATCHES" - values = [newrelic_alert_policy.alert.id] - } - } - - destination { - channel_id = newrelic_notification_channel.email-channel[0].id - } -} - -# Limitation of NR provider - only one workflow can be created per channel. Might be resolved in the future. -# https://registry.terraform.io/providers/newrelic/newrelic/latest/docs/resources/workflow#nested-destination-blocks -resource "newrelic_workflow" "alerting-workflow-slack" { - count = var.slack_enable ? 1 : 0 - name = format("%s slack alerting workflow", var.service_name) - muting_rules_handling = "NOTIFY_ALL_ISSUES" - - issues_filter { - name = format("%s alerting workflow filter", var.service_name) - type = "FILTER" - - predicate { - attribute = "labels.policyIds" - operator = "EXACTLY_MATCHES" - values = [newrelic_alert_policy.alert.id] - } - } - - destination { - channel_id = newrelic_notification_channel.slack-channel[0].id - } -} diff --git a/tf-managed/modules/daily-snapshot/monitoring/provider.tf b/tf-managed/modules/daily-snapshot/monitoring/provider.tf deleted file mode 100644 index 992d8225f..000000000 --- a/tf-managed/modules/daily-snapshot/monitoring/provider.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_version = "~> 1.6" - required_providers { - newrelic = { - source = "newrelic/newrelic" - version = "~> 3.0" - } - } -} diff --git a/tf-managed/modules/daily-snapshot/monitoring/variable.tf b/tf-managed/modules/daily-snapshot/monitoring/variable.tf deleted file mode 100644 index 76499aa60..000000000 --- a/tf-managed/modules/daily-snapshot/monitoring/variable.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "service_name" { - description = "The name of the service" - type = string -} - -variable "alert_email" { - description = "Email address to send alerts to" - type = string - default = "" -} - -variable "slack_enable" { - description = "Enable Slack notifications" - type = bool - default = false -} - -variable "slack_destination_id" { - description = "Slack destination id" - type = string -} - -variable "slack_channel_id" { - description = "Slack channel id" - type = string -} diff --git a/tf-managed/modules/daily-snapshot/outputs.tf b/tf-managed/modules/daily-snapshot/outputs.tf deleted file mode 100644 index 240c103f1..000000000 --- a/tf-managed/modules/daily-snapshot/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -# This ip address may be used in the future by monitoring software -output "ip" { - value = [digitalocean_droplet.forest.ipv4_address] -} diff --git a/tf-managed/modules/daily-snapshot/prep_sources.sh b/tf-managed/modules/daily-snapshot/prep_sources.sh deleted file mode 100755 index 1d63836c8..000000000 --- a/tf-managed/modules/daily-snapshot/prep_sources.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Enable strict error handling and command tracing -set -euxo pipefail - -# Copy local source files in a folder, and create a zip archive. -cd "$1" -rm -f sources.tar -(cd service && tar cf ../sources.tar --sort=name --mtime='UTC 2019-01-01' ./* > /dev/null 2>&1) -echo "{ \"path\": \"$1/sources.tar\" }" diff --git a/tf-managed/modules/daily-snapshot/provider.tf b/tf-managed/modules/daily-snapshot/provider.tf deleted file mode 100644 index c553a31e0..000000000 --- a/tf-managed/modules/daily-snapshot/provider.tf +++ /dev/null @@ -1,32 +0,0 @@ -terraform { - required_version = "~> 1.6" - - required_providers { - digitalocean = { - source = "digitalocean/digitalocean" - version = "~> 2.0" - } - external = { - source = "hashicorp/external" - version = "~> 2.1" - } - local = { - source = "hashicorp/local" - version = "~> 2.1" - } - newrelic = { - source = "newrelic/newrelic" - version = "~> 3.0" - } - } -} - -provider "digitalocean" { - token = var.digitalocean_token -} - -provider "newrelic" { - account_id = var.new_relic_account_id - api_key = var.new_relic_api_key - region = var.new_relic_region -} diff --git a/tf-managed/modules/daily-snapshot/service/calibnet_cron_job b/tf-managed/modules/daily-snapshot/service/calibnet_cron_job deleted file mode 100755 index 48ece9597..000000000 --- a/tf-managed/modules/daily-snapshot/service/calibnet_cron_job +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# shellcheck source=/dev/null -cd "$HOME" || exit -flock -n /tmp/calibnet.lock -c "docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env-file .forest_env -e NETWORK_CHAIN=calibnet ghcr.io/chainsafe/forest-snapshot-service:latest >> calibnet_log.txt 2>&1" diff --git a/tf-managed/modules/daily-snapshot/service/init.sh b/tf-managed/modules/daily-snapshot/service/init.sh deleted file mode 100755 index 1012c844c..000000000 --- a/tf-managed/modules/daily-snapshot/service/init.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -eux - -# Wait for cloud-init to finish initializing the machine -cloud-init status --wait - -# Setting DEBIAN_FRONTEND to ensure non-interactive operations for APT -export DEBIAN_FRONTEND=noninteractive - -# https://www.digitalocean.com/community/tutorials/how-to-set-up-time-synchronization-on-ubuntu-20-04 -# We will install the NTP service, so we need to disable the `systemd-timesyncd`. -# This is to prevent the two services from conflicting with one another. -timedatectl set-ntp no - -# Using timeout to ensure the script retries if the APT servers are temporarily unavailable. -timeout 10m bash -c 'until apt-get -qqq --yes update && \ - apt-get -qqq --yes install anacron ntp ; do sleep 10; \ -done' - -# Run new_relic and fail2ban scripts -bash newrelic_fail2ban.sh - -# Setup cron jobs -cp calibnet_cron_job mainnet_cron_job /etc/cron.hourly/ diff --git a/tf-managed/modules/daily-snapshot/service/mainnet_cron_job b/tf-managed/modules/daily-snapshot/service/mainnet_cron_job deleted file mode 100755 index b24ade470..000000000 --- a/tf-managed/modules/daily-snapshot/service/mainnet_cron_job +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# shellcheck source=/dev/null -cd "$HOME" || exit -flock -n /tmp/mainnet.lock -c "docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm --env-file .forest_env -e NETWORK_CHAIN=mainnet ghcr.io/chainsafe/forest-snapshot-service:latest >> mainnet_log.txt 2>&1" diff --git a/tf-managed/modules/daily-snapshot/service/newrelic_fail2ban.sh b/tf-managed/modules/daily-snapshot/service/newrelic_fail2ban.sh deleted file mode 100644 index 0e62ed350..000000000 --- a/tf-managed/modules/daily-snapshot/service/newrelic_fail2ban.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# This script configures New Relic infrastructure monitoring and Fail2Ban. -# It sets up the New Relic license key and custom configuration, adds the New Relic repository, -# refreshes it, and installs the New Relic infrastructure agent. -# It also installs Fail2Ban, sets up its default configuration, and enables it to start at boot - -set -euo pipefail -# If new relic API key is provided, install the new relic agent -if [ -n "$NEW_RELIC_API_KEY" ] ; then - curl -Ls https://download.newrelic.com/install/newrelic-cli/scripts/install.sh | bash && \ - sudo NEW_RELIC_API_KEY="$NEW_RELIC_API_KEY" \ - NEW_RELIC_ACCOUNT_ID="$NEW_RELIC_ACCOUNT_ID" \ - NEW_RELIC_REGION="$NEW_RELIC_REGION" \ - /usr/local/bin/newrelic install -y - -# The provided configurations are specific to New Relic. To gain a deeper understanding of these configuration details, you can visit: -# https://docs.newrelic.com/docs/infrastructure/install-infrastructure-agent/configuration/infrastructure-agent-configuration-settings/#offline-time-to-reset -cat >> /etc/newrelic-infra.yml < /etc/newrelic-infra/logging.d/logging.yml < -- - -The logic is contained in the [snapshot-age-monitor](./snapshot-age-monitor.js). - -Networks: `mainnet` and `calibnet` -Checks: -- asserts the epoch of the hosted snapshots is not older than a given threshold, -- trigger alarms if more than 2/3 checks over an hour fail. - -## Alarm destinations - -All alarms are handled via NewRelic, which integrates other applications. For now, it's: -* [PagerDuty](https://chainsafe.pagerduty.com/) - the provided destination is linked with a service generated there and visible in the [service direcory](https://chainsafe.pagerduty.com/service-directory). It will fire alarms to on-call engineers if enabled, so use it cautiously. -* [Slack](https://api.slack.com/) - the provided destination is also linked with company Slack, and the `channel_id` governs the actual channel to which the notifications will go. diff --git a/tf-managed/modules/snapshot-monitoring/main.tf b/tf-managed/modules/snapshot-monitoring/main.tf deleted file mode 100644 index 5d2db22d1..000000000 --- a/tf-managed/modules/snapshot-monitoring/main.tf +++ /dev/null @@ -1,146 +0,0 @@ -locals { - service_name = format("%s-snapshot-age-monitor", var.environment) -} - -resource "newrelic_alert_policy" "alert" { - name = format("%s alert policy", local.service_name) -} - -resource "newrelic_synthetics_script_monitor" "snapshot-age-monitor" { - status = "ENABLED" - name = format("%s-snapshot-age-monitor", var.environment) - type = "SCRIPT_API" - - # https://docs.newrelic.com/docs/synthetics/synthetic-monitoring/administration/synthetic-public-minion-ips/#public-minion-locations-and-location-labels-location - locations_public = ["AP_SOUTHEAST_1", "US_WEST_1", "EU_CENTRAL_1"] - period = "EVERY_HOUR" - script = file("snapshot-age-monitor.js") - - script_language = "JAVASCRIPT" - runtime_type = "NODE_API" - runtime_type_version = "16.10" - - tag { - key = "service" - values = ["forest-snapshot", var.environment] - } -} - -resource "newrelic_notification_channel" "slack-channel" { - count = var.slack_enable ? 1 : 0 - name = format("%s slack", local.service_name) - type = "SLACK" - destination_id = var.slack_destination_id - product = "IINT" - - property { - key = "channelId" - value = var.slack_channel_id - } - - property { - key = "customDetailsSlack" - value = "issue id - {{issueId}}" - } -} - -resource "newrelic_notification_channel" "pagerduty-channel" { - count = var.pagerduty_enable ? 1 : 0 - name = format("%s pagerduty", local.service_name) - type = "PAGERDUTY_SERVICE_INTEGRATION" - destination_id = var.pagerduty_destination_id - product = "IINT" - - property { - key = "summary" - value = "Filecoin Forest snapshot age monitor {{#isCritical}}CRITICAL{{/isCritical}}{{#isWarning}}WARNING{{/isWarning}}" - } - - property { - key = "customDetails" - value = <<-EOT - { - "id":{{json issueId}}, - "IssueURL":{{json issuePageUrl}}, - "NewRelic priority":{{json priority}}, - "Total Incidents":{{json totalIncidents}}, - "Impacted Entities":"{{#each entitiesData.names}}{{this}}{{#unless @last}}, {{/unless}}{{/each}}", - "Runbook":"{{#each accumulations.runbookUrl}}{{this}}{{#unless @last}}, {{/unless}}{{/each}}", - "Description":"{{#each annotations.description}}{{this}}{{#unless @last}}, {{/unless}}{{/each}}", - "isCorrelated":{{json isCorrelated}}, - "Alert Policy Names":"{{#each accumulations.policyName}}{{this}}{{#unless @last}}, {{/unless}}{{/each}}", - "Alert Condition Names":"{{#each accumulations.conditionName}}{{this}}{{#unless @last}}, {{/unless}}{{/each}}", - "Workflow Name":{{json workflowName}} - } - EOT - } -} - -resource "newrelic_nrql_alert_condition" "failing-snapshot-age" { - policy_id = newrelic_alert_policy.alert.id - name = format("%s failing snapshot age", local.service_name) - enabled = true - # Bound to the monitor query interval which is `EVERY_HOUR` - aggregation_window = 3600 - - nrql { - query = "SELECT filter(count(*), WHERE result = 'FAILED') AS 'Failures' FROM SyntheticCheck WHERE monitorName = '${local.service_name}'" - } - - # This means that all locations are failing - critical { - operator = "above_or_equals" - threshold = 3 - threshold_duration = 3600 - threshold_occurrences = "ALL" - } - - warning { - operator = "above_or_equals" - threshold = 2 - threshold_duration = 3600 - threshold_occurrences = "ALL" - } -} - -resource "newrelic_workflow" "alerting-workflow-slack" { - count = var.slack_enable ? 1 : 0 - name = format("%s slack alerting workflow", local.service_name) - muting_rules_handling = "NOTIFY_ALL_ISSUES" - - issues_filter { - name = format("%s alerting workflow filter", local.service_name) - type = "FILTER" - - predicate { - attribute = "labels.policyIds" - operator = "EXACTLY_MATCHES" - values = [newrelic_alert_policy.alert.id] - } - } - - destination { - channel_id = newrelic_notification_channel.slack-channel[0].id - } -} - -resource "newrelic_workflow" "alerting-workflow-pagerduty" { - count = var.pagerduty_enable ? 1 : 0 - name = format("%s pagerduty alerting workflow", local.service_name) - muting_rules_handling = "NOTIFY_ALL_ISSUES" - - issues_filter { - name = format("%s alerting workflow filter", local.service_name) - type = "FILTER" - - predicate { - attribute = "labels.policyIds" - operator = "EXACTLY_MATCHES" - values = [newrelic_alert_policy.alert.id] - } - } - - destination { - channel_id = newrelic_notification_channel.pagerduty-channel[0].id - } -} diff --git a/tf-managed/modules/snapshot-monitoring/provider.tf b/tf-managed/modules/snapshot-monitoring/provider.tf deleted file mode 100644 index 4cdacef6c..000000000 --- a/tf-managed/modules/snapshot-monitoring/provider.tf +++ /dev/null @@ -1,15 +0,0 @@ -terraform { - required_version = "~> 1.6" - required_providers { - newrelic = { - source = "newrelic/newrelic" - version = "~> 3.5" - } - } -} - -provider "newrelic" { - account_id = var.new_relic_account_id - api_key = var.new_relic_api_key - region = var.new_relic_region -} diff --git a/tf-managed/modules/snapshot-monitoring/snapshot-age-monitor.js b/tf-managed/modules/snapshot-monitoring/snapshot-age-monitor.js deleted file mode 100644 index 2bb01ea5a..000000000 --- a/tf-managed/modules/snapshot-monitoring/snapshot-age-monitor.js +++ /dev/null @@ -1,34 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/no-var-requires -- "approved" methods of resolving this lint do not work in the NR context. -var assert = require("assert"); - -function check_snapshot(url, genesisTime) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- that's how the callback works in this context. - var callback = function (_err, response, _body) { - assert.equal(response.statusCode, 200, "Expected a 200 OK response"); - - var snapshotName = response.url.split("/").pop(); - var height = snapshotName.match(/height_(\d+)/)[1]; - - var currentTime = Math.floor(Date.now() / 1000); - var snapshotTime = height * 30 + genesisTime; - var snapshotAgeInMinutes = (currentTime - snapshotTime) / 60; - - assert( - snapshotAgeInMinutes < 360, - "Expected snapshot to be less than 360 minutes old" - ); - }; - - // This variable is provided by New Relic. - // eslint-disable-next-line no-undef - $http.head(url, callback); -} - -check_snapshot( - "https://forest-internal.chainsafe.dev/latest/calibnet/", - 1667326380 -); -check_snapshot( - "https://forest-internal.chainsafe.dev/latest/mainnet/", - 1598306400 -); diff --git a/tf-managed/modules/snapshot-monitoring/variable.tf b/tf-managed/modules/snapshot-monitoring/variable.tf deleted file mode 100644 index 3729be5fc..000000000 --- a/tf-managed/modules/snapshot-monitoring/variable.tf +++ /dev/null @@ -1,64 +0,0 @@ -variable "environment" { - description = "The environment to deploy to" - type = string -} - -variable "new_relic_region" { - description = "The New Relic Platform Region" - type = string - default = "EU" -} - -variable "new_relic_api_key" { - description = "New Relic API KEY" - default = "" - type = string - sensitive = true -} - -variable "new_relic_account_id" { - description = "The New Relic Account ID" - default = 0 - type = number - sensitive = true -} - -variable "slack_enable" { - description = "Enable Slack notifications" - type = bool - default = false -} - -# This needs to be created manually. Afterwards, it can be found in -# NR / Alerts & AI / Destinations. -variable "slack_destination_id" { - description = "Slack destination id" - type = string - default = "" - sensitive = true -} - -# Channel ID - due to the limitations of NR it's not a human readable name -# but an ID. This can be found at the bottom of the channel settings window. -variable "slack_channel_id" { - description = "Slack channel id" - type = string - default = "" - sensitive = true -} - -variable "pagerduty_enable" { - description = "Enable PagerDuty notifications" - type = bool - default = false -} - -# This needs to be created manually. Afterwards, it can be found in -# NR / Alerts & AI / Destinations. Otherwise, we'd need special permissions -# to create it via the API. -variable "pagerduty_destination_id" { - description = "PagerDuty destination id" - type = string - default = "" - sensitive = true -}