From 12be84611d1d0d7d1e6db10b4a15a0e72057a716 Mon Sep 17 00:00:00 2001 From: sudo-shashank Date: Fri, 22 Mar 2024 13:52:25 +0530 Subject: [PATCH] Implement new archival service for lite and diff snapshots --- .../archival-snapshot/.terraform.lock.hcl | 42 +++++++++++++ tf-managed/modules/archival-snapshot/main.tf | 60 +++++++++++++++++++ .../modules/archival-snapshot/outputs.tf | 3 + .../modules/archival-snapshot/prep_sources.sh | 10 ++++ .../modules/archival-snapshot/provider.tf | 14 +++++ .../archival-snapshot/service/diff_script.sh | 24 ++++++++ .../modules/archival-snapshot/service/init.sh | 34 +++++++++++ .../archival-snapshot/service/upload_diff.sh | 14 +++++ .../modules/archival-snapshot/variable.tf | 15 +++++ 9 files changed, 216 insertions(+) create mode 100644 tf-managed/modules/archival-snapshot/.terraform.lock.hcl create mode 100644 tf-managed/modules/archival-snapshot/main.tf create mode 100644 tf-managed/modules/archival-snapshot/outputs.tf create mode 100755 tf-managed/modules/archival-snapshot/prep_sources.sh create mode 100644 tf-managed/modules/archival-snapshot/provider.tf create mode 100755 tf-managed/modules/archival-snapshot/service/diff_script.sh create mode 100755 tf-managed/modules/archival-snapshot/service/init.sh create mode 100755 tf-managed/modules/archival-snapshot/service/upload_diff.sh create mode 100644 tf-managed/modules/archival-snapshot/variable.tf diff --git a/tf-managed/modules/archival-snapshot/.terraform.lock.hcl b/tf-managed/modules/archival-snapshot/.terraform.lock.hcl new file mode 100644 index 000000000..72f07d78c --- /dev/null +++ b/tf-managed/modules/archival-snapshot/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.3" + constraints = "~> 2.1" + hashes = [ + "h1:H+3QlVPs/7CDa3I4KU/a23wYeGeJxeBlgvR7bfK1t1w=", + "zh:03d81462f9578ec91ce8e26f887e34151eda0e100f57e9772dbea86363588239", + "zh:37ec2a20f6a3ec3a0fd95d3f3de26da6cb9534b30488bc45723e118a0911c0d8", + "zh:4eb5b119179539f2749ce9de0e1b9629d025990f062f4f4dddc161562bb89d37", + "zh:5a31bb58414f41bee5e09b939012df5b88654120b0238a89dfd6691ba197619a", + "zh:6221a05e52a6a2d4f520ffe7cbc741f4f6080e0855061b0ed54e8be4a84eb9b7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8bb068496b4679bef625e4710d9f3432e301c3a56602271f04e60eadf7f8a94c", + "zh:94742aa5378bab626ce34f79bcef6a373e4f86ea7a8b762e9f71270a899e0d00", + "zh:a485831b5a525cd8f40e8982fa37da40ff70b1ae092c8b755fcde123f0b1238d", + "zh:a647ff16d071eabcabd87ea8183eb90a775a0294ddd735d742075d62fff09193", + "zh:b74710c5954aaa3faf262c18d36a8c2407862d9f842c63e7fa92fa4de3d29df6", + "zh:fa73d83edc92af2e551857594c2232ba6a9e3603ad34b0a5940865202c08d8d7", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.5.1" + constraints = "~> 2.1" + hashes = [ + "h1:8oTPe2VUL6E2d3OcrvqyjI4Nn/Y/UEQN26WLk5O/B0g=", + "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561", + "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561", + "zh:196bfb49086f22fd4db46033e01655b0e5e036a5582d250412cc690fa7995de5", + "zh:37c92ec084d059d37d6cffdb683ccf68e3a5f8d2eb69dd73c8e43ad003ef8d24", + "zh:4269f01a98513651ad66763c16b268f4c2da76cc892ccfd54b401fff6cc11667", + "zh:51904350b9c728f963eef0c28f1d43e73d010333133eb7f30999a8fb6a0cc3d8", + "zh:73a66611359b83d0c3fcba2984610273f7954002febb8a57242bbb86d967b635", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7ae387993a92bcc379063229b3cce8af7eaf082dd9306598fcd42352994d2de0", + "zh:9e0f365f807b088646db6e4a8d4b188129d9ebdbcf2568c8ab33bddd1b82c867", + "zh:b5263acbd8ae51c9cbffa79743fbcadcb7908057c87eb22fd9048268056efbc4", + "zh:dfcd88ac5f13c0d04e24be00b686d069b4879cc4add1b7b1a8ae545783d97520", + ] +} diff --git a/tf-managed/modules/archival-snapshot/main.tf b/tf-managed/modules/archival-snapshot/main.tf new file mode 100644 index 000000000..430b22897 --- /dev/null +++ b/tf-managed/modules/archival-snapshot/main.tf @@ -0,0 +1,60 @@ +# This terraform script executes the following steps: +# - Sync with latest lite+diff snapshots. +# - Export a large master snapshot covering, say, 30,000 epochs. +# - Generate diffs and lite snapshots from the master snapshot. Eg. forest-tool archive export ... +# - Upload generated snapshots. + +# Ugly hack because 'archive_file' cannot mix files and folders. +data "external" "sources_tar" { + program = ["bash", "${path.module}/prep_sources.sh", path.module] +} + +data "local_file" "sources" { + filename = data.external.sources_tar.result.path +} + +data "local_file" "init" { + filename = "${path.module}/service/init.sh" +} + +# Required environment variables for the service. +locals { + env_content = <<-EOT + SLACK_TOKEN=${var.slack_token} + SLACK_CHANNEL=${var.slack_channel} + EOT +} + +resource "null_resource" "copy_and_execute" { + depends_on = [data.external.sources_tar] + + provisioner "file" { + source = data.external.sources_tar.result.path + destination = "/tmp/sources.tar" + + connection { + type = "ssh" + host = "archie.chainsafe.dev" + user = "archie" + private_key = file("~/.ssh/id_rsa") + } + } + + provisioner "remote-exec" { + inline = [ + "mkdir -p /mnt/md0/exported/archival", + "mv /tmp/sources.tar /mnt/md0/exported/archival/sources.tar", + "cd /mnt/md0/exported/archival", + "tar xf sources.tar", + "nohup sh ./init.sh > ./init_log.txt &", + "sleep 60s", + ] + + connection { + type = "ssh" + host = "archie.chainsafe.dev" + user = "archie" + private_key = file("~/.ssh/id_rsa") + } + } +} diff --git a/tf-managed/modules/archival-snapshot/outputs.tf b/tf-managed/modules/archival-snapshot/outputs.tf new file mode 100644 index 000000000..4c3e8747a --- /dev/null +++ b/tf-managed/modules/archival-snapshot/outputs.tf @@ -0,0 +1,3 @@ +output "ssh_hostname" { + value = "archie.chainsafe.dev" +} diff --git a/tf-managed/modules/archival-snapshot/prep_sources.sh b/tf-managed/modules/archival-snapshot/prep_sources.sh new file mode 100755 index 000000000..1d63836c8 --- /dev/null +++ b/tf-managed/modules/archival-snapshot/prep_sources.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Enable strict error handling and command tracing +set -euxo pipefail + +# Copy local source files in a folder, and create a zip archive. +cd "$1" +rm -f sources.tar +(cd service && tar cf ../sources.tar --sort=name --mtime='UTC 2019-01-01' ./* > /dev/null 2>&1) +echo "{ \"path\": \"$1/sources.tar\" }" diff --git a/tf-managed/modules/archival-snapshot/provider.tf b/tf-managed/modules/archival-snapshot/provider.tf new file mode 100644 index 000000000..1b7a17cd8 --- /dev/null +++ b/tf-managed/modules/archival-snapshot/provider.tf @@ -0,0 +1,14 @@ +terraform { + required_version = "~> 1.6" + + required_providers { + external = { + source = "hashicorp/external" + version = "~> 2.1" + } + local = { + source = "hashicorp/local" + version = "~> 2.1" + } + } +} diff --git a/tf-managed/modules/archival-snapshot/service/diff_script.sh b/tf-managed/modules/archival-snapshot/service/diff_script.sh new file mode 100755 index 000000000..54b034444 --- /dev/null +++ b/tf-managed/modules/archival-snapshot/service/diff_script.sh @@ -0,0 +1,24 @@ +#!/bin/env bash + +set -euxo pipefail + +FOREST=~/forest-v0.14.0/forest-tool + +EPOCH_START="$1" +shift +DIFF_STEP=3000 +DIFF_COUNT=10 +GENESIS_TIMESTAMP=1598306400 +SECONDS_PER_EPOCH=30 + +for i in `seq 1 $DIFF_COUNT`; do + EPOCH=$((EPOCH_START+DIFF_STEP*i)) + EPOCH_TIMESTAMP=$((1598306400 + EPOCH*SECONDS_PER_EPOCH)) + DATE=$(date --date=@$EPOCH_TIMESTAMP -u -I) + FILE=diff_snapshots/forest_diff_mainnet_"$DATE"_height_$((EPOCH-DIFF_STEP))+"$DIFF_STEP".forest.car.zst + if ! test -f $FILE; then + "$FOREST" archive export --depth $DIFF_STEP --epoch $EPOCH --diff $((EPOCH-DIFF_STEP)) --diff-depth 900 --output-path $FILE $@ + else + echo Skipping $FILE + fi +done diff --git a/tf-managed/modules/archival-snapshot/service/init.sh b/tf-managed/modules/archival-snapshot/service/init.sh new file mode 100755 index 000000000..4d2326c1e --- /dev/null +++ b/tf-managed/modules/archival-snapshot/service/init.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +## Enable strict error handling, command tracing, and pipefail +set -euxo pipefail + +ENDPOINT="$1" +FOREST_CLI="/home/archie/forest-v0.16.3/forest-cli" +FOREST_TOOL="/home/archie/forest-v0.16.3/forest-tool" + +# Setting DEBIAN_FRONTEND to ensure non-interactive operations for APT +export DEBIAN_FRONTEND=noninteractive + +CURRENT_SNAPSHOT=$(aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/lite/" | sort | tail -n 1 | awk '{print $NF}') +CURRENT_EPOCH=$(echo "$CURRENT_SNAPSHOT" | awk -F'_' '{gsub(/[^0-9]/, "", $6); print $6}') + +echo "$CURRENT_EPOCH" + +LATEST_EPOCH=$($FOREST_CLI sync status | grep "Height:" | awk '{print $2}') +echo "$LATEST_EPOCH" + +while ((LATEST_EPOCH - CURRENT_EPOCH > 30000)); do + NEW_EPOCH=$((CURRENT_EPOCH + 30000)) + NEW_SNAPSHOT=$($FOREST_CLI snapshot export --tipset "$NEW_EPOCH" --depth 30000 | grep forest |awk -F'[:]' '{print $1}') + + # Generate and upload lite snapshot + $FOREST_TOOL archive export --epoch "$NEW_EPOCH" --output-path ./lite_snapshots/ "$NEW_SNAPSHOT" + aws --profile prod --endpoint "$ENDPOINT" s3 cp ./lite_snapshots/"$NEW_SNAPSHOT" "s3://forest-archive/mainnet/lite/" + + # Generate and upload diff snapshots + ./diff_script.sh "$CURRENT_EPOCH" "$CURRENT_SNAPSHOT" "$NEW_SNAPSHOT" + ./upload_diff.sh "$CURRENT_EPOCH" + + CURRENT_EPOCH=$NEW_EPOCH +done diff --git a/tf-managed/modules/archival-snapshot/service/upload_diff.sh b/tf-managed/modules/archival-snapshot/service/upload_diff.sh new file mode 100755 index 000000000..95cf7c502 --- /dev/null +++ b/tf-managed/modules/archival-snapshot/service/upload_diff.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +ENDPOINT="$1" + +# list files to upload +ls diff_snapshots/ > upload_files.txt + +while read -r file; do + # Upload the file to the S3 bucket + aws --profile prod --endpoint "$ENDPOINT" s3 cp diff_snapshots/"$file" "s3://forest-archive/mainnet/diff/" +done < upload_files.txt + +# Remove uploaded diff snapshots +rm diff_snapshots/* -rf diff --git a/tf-managed/modules/archival-snapshot/variable.tf b/tf-managed/modules/archival-snapshot/variable.tf new file mode 100644 index 000000000..59f0bef70 --- /dev/null +++ b/tf-managed/modules/archival-snapshot/variable.tf @@ -0,0 +1,15 @@ +variable "name" { + description = "The name of the server" + type = string +} + +variable "slack_channel" { + description = "Slack channel name for notifications" + type = string +} + +variable "slack_token" { + description = "Slack access token" + type = string + sensitive = true +}