diff --git a/.github/workflows/deploy-daily-snapshot.yml b/.github/workflows/deploy-daily-snapshot.yml index 9a4877b78..224be8b77 100644 --- a/.github/workflows/deploy-daily-snapshot.yml +++ b/.github/workflows/deploy-daily-snapshot.yml @@ -39,7 +39,7 @@ jobs: r2_access_key: ${{ secrets.R2_ACCESS_KEY }} r2_secret_key: ${{ secrets.R2_SECRET_KEY }} slack_token: ${{ secrets.SLACK_TOKEN }} - working_directory: terraform/daily_snapshot + working_directory: terraform/daily_snapshot/prod environment: Snapshot Service new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }} diff --git a/terraform/daily_snapshot/README.md b/terraform/daily_snapshot/README.md index 6fa5fa0dd..15dafcf89 100644 --- a/terraform/daily_snapshot/README.md +++ b/terraform/daily_snapshot/README.md @@ -1,6 +1,9 @@ # Overview -This directory contains an infrastructure configuration for the Forest service, which generates daily snapshots. This configuration ensures the continuous verification of Forest's ability to export snapshots. Once a day, the service synchronizes with Calibnet and creates a new snapshot. If the previous snapshot is older than one day, the new snapshot is uploaded to Digital Ocean Spaces. Additionally, the New Relic Infrastructure agent is installed to facilitate monitoring +This directory contains an infrastructure configuration for the Forest service, +which generates the latest snapshots available at this endpoint: +https://forest-archive.chainsafe.dev/latest/mainnet/ and +https://forest-archive.chainsafe.dev/latest/calibnet/ # Workflow @@ -37,60 +40,20 @@ export AWS_SECRET_ACCESS_KEY= # Cloudflare R2 secret access keys used by the snapshot service. export TF_VAR_R2_ACCESS_KEY= export TF_VAR_R2_SECRET_KEY= - -# Optional, only if you want install new relic agent -# New Relic License key, Can be generated here: https://one.eu.newrelic.com/admin-portal/api-keys/home -export TF_VAR_NEW_RELIC_API_KEY= -export TF_VAR_NEW_RELIC_ACCOUNT_ID= ``` -Forest tokens can be found on 1password (for production deployments only; for non-production deployments, generate your own secrets). - -you'll also need to link your public key with Digital Ocean. To do this, visit https://cloud.digitalocean.com/account/security. Additionally, set up your SSH key by following the commands provided below: - -```bash -eval $(ssh-agent) - -ssh-add -``` - -To ensure the production Snapshot service remains intact, modify certain variables in the `Main.tf` file: - -- Change `key = "daily_snapshot.tfstate"` to `key = ".tfstate"`. -- Replace `name = "forest-snapshot"` with `name = ""`. -- Replace ` slack_channel = "#forest-notifications"` with `slack_channel = "#forest-dump"` -- Replace `snapshot_bucket = "forest-snapshots"` with `snapshot_bucket = ""` - -Remember to replace ``, ``, ``, and `` with appropriate values. - -Note that there is conditional logic for the Slack messages (i.e., only report a successful snapshot upload if this is the first new snapshot of the day), so--depending on how you populated your custom snapshot bucket and when you are running the deployment--you may or may not see a success message in the `#forest-dump` channel without manually modifying this logic. - -To prepare terraform for other commands: +Forest tokens can be found on 1password. -```bash -$ terraform init -``` - -To inspect a new deployment plan (it'll tell you which servers will be removed, -added, etc.): +Playbook: ```bash -$ terraform plan +$ terraform init # Initialize terraform state +$ terraform plan # Show deployment plan (optional) +$ terraform apply # Apply deployment plan +$ terraform destroy # Destroy deployment ``` For Mac users, if you encounter the `Error: External Program Execution Failed`, you'll need to adjust the `prep_sources.sh` file located in the `../modules/daily_snapshot` directory. Make the following changes: - Replace `--archive` with `-Rp`. - Install `gnu-tar` using the command `brew install gnu-tar`. Afterward, switch `tar cf ../sources.tar` to `gtar cf ../sources.tar` - -To deploy the service: - -```bash -$ terraform apply -``` - -To shutdown the service: - -```bash -$ terraform destroy -``` diff --git a/terraform/daily_snapshot/main.tf b/terraform/daily_snapshot/main.tf index 9ff0cf8b2..042b9603b 100644 --- a/terraform/daily_snapshot/main.tf +++ b/terraform/daily_snapshot/main.tf @@ -8,7 +8,7 @@ terraform { # This key uniquely identifies the service. To create a new service (instead # of modifying this one), use a new key. Unfortunately, variables may not be # used here. - key = "daily_snapshot.tfstate" + key = "daily_snapshot_dev.tfstate" # This value is completely unused by DO but _must_ be a known AWS region. region = "us-west-1" @@ -28,10 +28,10 @@ module "daily_snapshot" { source = "../modules/daily_snapshot" # Configure service: - name = "forest-snapshot" # droplet name - size = "s-4vcpu-16gb-amd" # droplet size - slack_channel = "#forest-notifications" # slack channel for notifications - snapshot_bucket = "forest-snapshots" + name = "forest-snapshot-dev" # droplet name + size = "s-4vcpu-16gb-amd" # droplet size + slack_channel = "#forest-dump" # slack channel for notifications + snapshot_bucket = "forest-archive-dev" snapshot_endpoint = "fra1.digitaloceanspaces.com" forest_tag = "latest" r2_endpoint = "https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com/" @@ -41,8 +41,8 @@ module "daily_snapshot" { R2_ACCESS_KEY = var.R2_ACCESS_KEY R2_SECRET_KEY = var.R2_SECRET_KEY digitalocean_token = var.do_token - NEW_RELIC_API_KEY = var.NEW_RELIC_API_KEY - NEW_RELIC_ACCOUNT_ID = var.NEW_RELIC_ACCOUNT_ID + NEW_RELIC_API_KEY = "" + NEW_RELIC_ACCOUNT_ID = "" } # This ip address may be used in the future by monitoring software diff --git a/terraform/daily_snapshot/prod/main.tf b/terraform/daily_snapshot/prod/main.tf new file mode 100644 index 000000000..adce11d70 --- /dev/null +++ b/terraform/daily_snapshot/prod/main.tf @@ -0,0 +1,50 @@ +terraform { + required_version = "~> 1.3" + + backend "s3" { + # Note: This is the bucket for the internal terraform state. This bucket is + # completely independent from the bucket that contains snapshots. + bucket = "forest-iac" + # This key uniquely identifies the service. To create a new service (instead + # of modifying this one), use a new key. Unfortunately, variables may not be + # used here. + key = "daily_snapshot.tfstate" + + # This value is completely unused by DO but _must_ be a known AWS region. + region = "us-west-1" + # The S3 region is determined by the endpoint. fra1 = Frankfurt. + # This region does not have to be shared by the droplet. + endpoint = "https://fra1.digitaloceanspaces.com" + + # Credentially can be validated through the Security Token Service (STS). + # Unfortunately, DigitalOcean does not support STS so we have to skip the + # validation. + skip_credentials_validation = "true" + } +} + +module "daily_snapshot" { + # Import the daily_snapshot module + source = "../../modules/daily_snapshot" + + # Configure service: + name = "forest-snapshot" # droplet name + size = "s-4vcpu-16gb-amd" # droplet size + slack_channel = "#forest-notifications" # slack channel for notifications + snapshot_bucket = "forest-archive" + forest_tag = "latest" + r2_endpoint = "https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com/" + + # Variable passthrough: + slack_token = var.slack_token + R2_ACCESS_KEY = var.R2_ACCESS_KEY + R2_SECRET_KEY = var.R2_SECRET_KEY + digitalocean_token = var.do_token + NEW_RELIC_API_KEY = var.NEW_RELIC_API_KEY + NEW_RELIC_ACCOUNT_ID = var.NEW_RELIC_ACCOUNT_ID +} + +# This ip address may be used in the future by monitoring software +output "ip" { + value = [module.daily_snapshot.ip] +} diff --git a/terraform/daily_snapshot/prod/variable.tf b/terraform/daily_snapshot/prod/variable.tf new file mode 100644 index 000000000..b445458eb --- /dev/null +++ b/terraform/daily_snapshot/prod/variable.tf @@ -0,0 +1,29 @@ +variable "do_token" { + description = "Token for authentication." + type = string +} + +variable "R2_ACCESS_KEY" { + description = "S3 access key id" + type = string +} + +variable "R2_SECRET_KEY" { + description = "S3 private access key" + type = string +} + +variable "slack_token" { + description = "slack access token" + type = string +} + +variable "NEW_RELIC_API_KEY" { + description = "New Relic API KEY" + type = string +} + +variable "NEW_RELIC_ACCOUNT_ID" { + description = "The New Relic Account ID" + type = string +} diff --git a/terraform/daily_snapshot/variable.tf b/terraform/daily_snapshot/variable.tf index b445458eb..cac709952 100644 --- a/terraform/daily_snapshot/variable.tf +++ b/terraform/daily_snapshot/variable.tf @@ -17,13 +17,3 @@ variable "slack_token" { description = "slack access token" type = string } - -variable "NEW_RELIC_API_KEY" { - description = "New Relic API KEY" - type = string -} - -variable "NEW_RELIC_ACCOUNT_ID" { - description = "The New Relic Account ID" - type = string -} diff --git a/terraform/modules/daily_snapshot/service/daily_snapshot.rb b/terraform/modules/daily_snapshot/service/daily_snapshot.rb index ea4531048..fa8e3ff29 100644 --- a/terraform/modules/daily_snapshot/service/daily_snapshot.rb +++ b/terraform/modules/daily_snapshot/service/daily_snapshot.rb @@ -12,8 +12,6 @@ BASE_FOLDER = get_and_assert_env_variable 'BASE_FOLDER' SLACK_TOKEN = get_and_assert_env_variable 'SLACK_API_TOKEN' CHANNEL = get_and_assert_env_variable 'SLACK_NOTIF_CHANNEL' -BUCKET = get_and_assert_env_variable 'SNAPSHOT_BUCKET' -ENDPOINT = get_and_assert_env_variable 'SNAPSHOT_ENDPOINT' # Query the date of the most recent snapshot. def latest_snapshot_date(chain_name = 'calibnet') diff --git a/terraform/modules/daily_snapshot/service/forest-env.tpl b/terraform/modules/daily_snapshot/service/forest-env.tpl index f919600bb..14f644c7a 100644 --- a/terraform/modules/daily_snapshot/service/forest-env.tpl +++ b/terraform/modules/daily_snapshot/service/forest-env.tpl @@ -4,7 +4,6 @@ export R2_ENDPOINT="${r2_endpoint}" export SLACK_API_TOKEN="${slack_token}" export SLACK_NOTIF_CHANNEL="${slack_channel}" export SNAPSHOT_BUCKET="${snapshot_bucket}" -export SNAPSHOT_ENDPOINT="${snapshot_endpoint}" export NEW_RELIC_API_KEY="${NEW_RELIC_API_KEY}" export NEW_RELIC_ACCOUNT_ID="${NEW_RELIC_ACCOUNT_ID}" export NEW_RELIC_REGION="${NEW_RELIC_REGION}" diff --git a/terraform/modules/daily_snapshot/service/upload_filops_snapshot.sh b/terraform/modules/daily_snapshot/service/upload_filops_snapshot.sh index a4ec840b8..faa540dd1 100644 --- a/terraform/modules/daily_snapshot/service/upload_filops_snapshot.sh +++ b/terraform/modules/daily_snapshot/service/upload_filops_snapshot.sh @@ -50,7 +50,7 @@ if [ ${DIFF} -gt 1 ]; then ghcr.io/chainsafe/forest:"${FOREST_TAG}" \ -c "$COMMANDS" || exit 1 - if aws --endpoint "$R2_ENDPOINT" s3 cp "$BASE_FOLDER/forest_db/filops/filops_snapshot_$CHAIN"* s3://forest-archive/"$CHAIN_NAME"/latest/; then + if aws --endpoint "$R2_ENDPOINT" s3 cp "$BASE_FOLDER/forest_db/filops/filops_snapshot_$CHAIN"* s3://"$SNAPSHOT_BUCKET"/"$CHAIN_NAME"/latest/; then # Send alert to Slack only if upload is successful send_slack_alert "Old $CHAIN snapshot detected. 🔥🌲🔥. Filops Snapshot upload successful:✅" rm "$BASE_FOLDER/forest_db/filops/filops_snapshot_$CHAIN"*