Implement new archival service for lite and diff snapshots

tf-managed/modules/archival-snapshot/main.tf

@@ -0,0 +1,60 @@
+# This terraform script executes the following steps:
+#  - Sync with latest lite+diff snapshots.
+#  - Export a large master snapshot covering, say, 30,000 epochs.
+#  - Generate diffs and lite snapshots from the master snapshot. Eg. forest-tool archive export ...
+#  - Upload generated snapshots.
+
+# Ugly hack because 'archive_file' cannot mix files and folders.
+data "external" "sources_tar" {
+  program = ["bash", "${path.module}/prep_sources.sh", path.module]
+}
+
+data "local_file" "sources" {
+  filename = data.external.sources_tar.result.path
+}
+
+data "local_file" "init" {
+  filename = "${path.module}/service/init.sh"
+}
+
+# Required environment variables for the service.
+locals {
+  env_content = <<-EOT
+    SLACK_TOKEN=${var.slack_token}
+    SLACK_CHANNEL=${var.slack_channel}
+  EOT
+}
+
+resource "null_resource" "copy_and_execute" {
+  depends_on = [data.external.sources_tar]
+
+  provisioner "file" {
+    source      = data.external.sources_tar.result.path
+    destination = "/tmp/sources.tar"
+
+    connection {
+      type        = "ssh"
+      host        = "archie.chainsafe.dev"
+      user        = "archie"
+      private_key = file("~/.ssh/id_rsa")
+    }
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "mkdir -p /mnt/md0/exported/archival",
+      "mv /tmp/sources.tar /mnt/md0/exported/archival/sources.tar",
+      "cd /mnt/md0/exported/archival",
+      "tar xf sources.tar",
+      "nohup sh ./init.sh > ./init_log.txt &",
+      "sleep 60s",
+    ]
+
+    connection {
+      type        = "ssh"
+      host        = "archie.chainsafe.dev"
+      user        = "archie"
+      private_key = file("~/.ssh/id_rsa")
+    }
+  }
+}

tf-managed/modules/archival-snapshot/outputs.tf

@@ -0,0 +1,3 @@
+output "ssh_hostname" {
+  value = "archie.chainsafe.dev"
+}

tf-managed/modules/archival-snapshot/prep_sources.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Enable strict error handling and command tracing
+set -euxo pipefail
+
+# Copy local source files in a folder, and create a zip archive.
+cd "$1"
+rm -f sources.tar
+(cd service && tar cf ../sources.tar --sort=name --mtime='UTC 2019-01-01' ./* > /dev/null 2>&1)
+echo "{ \"path\": \"$1/sources.tar\" }"

tf-managed/modules/archival-snapshot/provider.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_version = "~> 1.6"
+
+  required_providers {
+    external = {
+      source  = "hashicorp/external"
+      version = "~> 2.1"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = "~> 2.1"
+    }
+  }
+}

tf-managed/modules/archival-snapshot/service/diff_script.sh

@@ -0,0 +1,24 @@
+#!/bin/env bash
+
+set -euxo pipefail
+
+FOREST=~/forest-v0.14.0/forest-tool
+
+EPOCH_START="$1"
+shift
+DIFF_STEP=3000
+DIFF_COUNT=10
+GENESIS_TIMESTAMP=1598306400
+SECONDS_PER_EPOCH=30
+
+for i in `seq 1 $DIFF_COUNT`; do
+    EPOCH=$((EPOCH_START+DIFF_STEP*i))
+    EPOCH_TIMESTAMP=$((1598306400 + EPOCH*SECONDS_PER_EPOCH))
+    DATE=$(date --date=@$EPOCH_TIMESTAMP -u -I)
+    FILE=diff_snapshots/forest_diff_mainnet_"$DATE"_height_$((EPOCH-DIFF_STEP))+"$DIFF_STEP".forest.car.zst
+    if ! test -f $FILE; then
+        "$FOREST" archive export --depth $DIFF_STEP --epoch $EPOCH --diff $((EPOCH-DIFF_STEP)) --diff-depth 900 --output-path $FILE $@
+    else
+        echo Skipping $FILE
+    fi
+done

tf-managed/modules/archival-snapshot/service/init.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+## Enable strict error handling, command tracing, and pipefail
+set -euxo pipefail
+
+ENDPOINT="$1"
+FOREST_CLI="/home/archie/forest-v0.16.3/forest-cli"
+FOREST_TOOL="/home/archie/forest-v0.16.3/forest-tool"
+
+# Setting DEBIAN_FRONTEND to ensure non-interactive operations for APT
+export DEBIAN_FRONTEND=noninteractive
+
+CURRENT_SNAPSHOT=$(aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/lite/" | sort | tail -n 1 | awk '{print $NF}')
+CURRENT_EPOCH=$(echo "$CURRENT_SNAPSHOT" | awk -F'_' '{gsub(/[^0-9]/, "", $6); print $6}')
+
+echo "$CURRENT_EPOCH"
+
+LATEST_EPOCH=$($FOREST_CLI sync status | grep "Height:" | awk '{print $2}')
+echo "$LATEST_EPOCH"
+
+while ((LATEST_EPOCH - CURRENT_EPOCH > 30000)); do
+   NEW_EPOCH=$((CURRENT_EPOCH + 30000))
+   NEW_SNAPSHOT=$($FOREST_CLI snapshot export --tipset "$NEW_EPOCH" --depth 30000 | grep forest |awk -F'[:]' '{print $1}')
+
+   # Generate and upload lite snapshot
+   $FOREST_TOOL archive export --epoch "$NEW_EPOCH" --output-path ./lite_snapshots/ "$NEW_SNAPSHOT"
+   aws --profile prod --endpoint "$ENDPOINT" s3 cp ./lite_snapshots/"$NEW_SNAPSHOT" "s3://forest-archive/mainnet/lite/"
+
+   # Generate and upload diff snapshots
+   ./diff_script.sh "$CURRENT_EPOCH" "$CURRENT_SNAPSHOT" "$NEW_SNAPSHOT"
+   ./upload_diff.sh "$CURRENT_EPOCH"
+
+   CURRENT_EPOCH=$NEW_EPOCH
+done

tf-managed/modules/archival-snapshot/service/upload_diff.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ENDPOINT="$1"
+
+# list files to upload
+ls diff_snapshots/ > upload_files.txt
+
+while read -r file; do
+    # Upload the file to the S3 bucket
+    aws --profile prod --endpoint "$ENDPOINT" s3 cp diff_snapshots/"$file" "s3://forest-archive/mainnet/diff/"
+done < upload_files.txt
+
+# Remove uploaded diff snapshots 
+rm diff_snapshots/* -rf

tf-managed/modules/archival-snapshot/variable.tf

@@ -0,0 +1,15 @@
+variable "name" {
+  description = "The name of the server"
+  type        = string
+}
+
+variable "slack_channel" {
+  description = "Slack channel name for notifications"
+  type        = string
+}
+
+variable "slack_token" {
+  description = "Slack access token"
+  type        = string
+  sensitive   = true
+}