Merge pull request #1860 from habitat-sh/jah/migrate_minio

minio migration via hooks

components/builder-minio/habitat/config_install/minio-migration.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+function download_bucket_objects() {
+  echo "DOWNLOADING objects from the MinIO that we are migrating from"
+  if ! aws --endpoint-url "$MINIO_ENDPOINT" s3 sync "$s3_url" "$WAYPOINT"; then
+    echo "ERROR: download of objects FAILED"
+    exit 8
+  fi
+  return 0
+}
+
+function _ensure_bucket_exists() {
+  if ! aws --endpoint-url "$MINIO_ENDPOINT" s3 ls "{{cfg.bucket_name}}" &>/dev/null; then
+    aws --endpoint-url "$MINIO_ENDPOINT" s3 mb "$s3_url"
+  fi
+}
+
+function upload_bucket_objects() {
+  _ensure_bucket_exists
+  echo "UPLOADING objects to the MinIO that we are migrating to"
+  if ! aws --endpoint-url "$MINIO_ENDPOINT" s3 sync "$WAYPOINT" "$s3_url"; then
+    echo "ERROR: upload of objects FAILED"
+    exit 9
+  fi
+  return 0
+}
+
+function is_migration_from_removed_fs_backend_needed() {
+  echo "CHECKING if migration from removed fs backend is needed"
+  if [[ -f /hab/svc/builder-minio/data/.minio.sys/format.json ]]; then
+    format_value=$(jq -r '.format' /hab/svc/builder-minio/data/.minio.sys/format.json)
+    if [[ "${format_value}" == 'fs' ]]; then
+      return 0
+    fi
+  fi
+  return 1
+}
+
+function minio_health_live_check() {
+  for ((n = 0; n < 20; n++)); do
+    curl_http_code_args=(-fs -o /dev/null -w "%{http_code}" --retry 4 --retry-delay 1)
+    code=$(curl "${curl_http_code_args[@]}" "$MINIO_ENDPOINT/minio/health/live")
+    if [[ $code == 200 ]]; then
+      return 0
+    else
+      sleep .5
+    fi
+  done
+  return 1
+}
+
+function minio_stop() {
+  for ((n = 0; n < 20; n++)); do
+    if pgrep minio &>/dev/null; then
+      pkill minio &>/dev/null
+      sleep 1
+    else
+      return 0
+    fi
+  done
+}
+
+function config_environment_for_migration() {
+  if [[ ! $1 =~ ^[0-9]+$ ]]; then
+    echo "ERROR: Invalid timestamp"
+  fi
+  if [ -f "{{pkg.svc_files_path}}/private.key" ]; then
+    MINIO_ENDPOINT="https://localhost:{{cfg.bind_port}}"
+  else
+    MINIO_ENDPOINT="http://localhost:{{cfg.bind_port}}"
+  fi
+  export MINIO_ENDPOINT
+  export AWS_ACCESS_KEY_ID="{{cfg.env.MINIO_ACCESS_KEY}}"
+  export AWS_SECRET_ACCESS_KEY="{{cfg.env.MINIO_SECRET_KEY}}"
+  export s3_url="s3://{{cfg.bucket_name}}"
+  WAYPOINT=$(mktemp -d -t minio-waypoint-"$1"-XXXXXXXXXX)
+  export WAYPOINT
+}
+
+function _enumerate_bucket_objects() {
+  aws --endpoint-url "$MINIO_ENDPOINT" s3 ls "$s3_url" --recursive --summarize >"$1"
+}
+
+function summarize_old_minio_bucket_objects() {
+  if [[ ! $1 =~ ^[0-9]+$ ]]; then
+    echo "Invalid timestamp"
+  fi
+  local tempfile
+  tempfile=$(mktemp -t "minio-old-contents-summary-$1-XXXXXXXXXX")
+  _enumerate_bucket_objects "$tempfile"
+  echo "$tempfile"
+}
+
+function summarize_new_minio_bucket_objects() {
+  if [[ ! $1 =~ ^[0-9]+$ ]]; then
+    echo "ERROR: Invalid timestamp"
+  fi
+  local tempfile
+  tempfile=$(mktemp -t "minio-new-contents-summary-$1-XXXXXXXXXX")
+  _enumerate_bucket_objects "$tempfile"
+  echo "$tempfile"
+}

components/builder-minio/habitat/default.toml

@@ -36,7 +36,11 @@ bind_port = 9000
 # Doc: https://docs.min.io/docs/minio-server-configuration-guide.html
 [env]
 # Set minio admin user and password
+# MINIO_ACCESS_KEY and MINIO_SECRET_KEY are deprecated.
+# Use MINIO_ROOT_USER and MINIO_ROOT_PASSWORD
+MINIO_ROOT_USER = "depot"
 MINIO_ACCESS_KEY = "depot"
+MINIO_ROOT_PASSWORD = "password"
 MINIO_SECRET_KEY = "password"
 
 # Specify private key password for TLS mode

components/builder-minio/habitat/hooks/install

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+exec 2>&1
+set -euo pipefail
+
+# shellcheck disable=SC1091
+source "{{pkg.svc_config_install_path}}/minio-migration.sh"
+
+timestamp=$EPOCHSECONDS # provides a shared element in naming items below
+config_environment_for_migration "$timestamp"
+
+echo "Checking if MinIO migration is needed"
+if is_migration_from_removed_fs_backend_needed; then
+
+  echo "BEGIN MinIO migration"
+
+  if pgrep minio &>/dev/null; then
+
+    if ! minio_health_live_check; then
+      echo "There seems to be a MinIO process but it fails the health check"
+      exit 9
+    fi
+
+  else
+
+    minio_pkg_old="core/minio/2018-10-05T01-03-03Z/20181006010122"
+    # This pkg_id is the minio dependency used in the on-prem-stable version of
+    # https://bldr.habitat.sh/#/pkgs/habitat/builder-minio/7764/20181006010221
+    # which was on-prem-stable for 6+ years so this should be a safe choice
+
+    if ! hab pkg list $minio_pkg_old >/dev/null; then
+      echo "ERROR: $minio_pkg_old is not installed and we require it for the migration."
+      exit 11
+    fi
+
+    old_minio_stdout=$(mktemp -t "minio-old-stdout-$timestamp-XXXXXXXXXX")
+    hab pkg exec $minio_pkg_old minio -- server \
+      --config-dir "{{pkg.svc_config_path}}" \
+      /hab/svc/builder-minio/data &>"$old_minio_stdout" &
+
+    if ! minio_health_live_check; then
+      echo "MinIO is not running so MinIO migration cannot begin"
+      exit 10
+    fi
+
+  fi
+
+  minio_old_summary=$(summarize_old_minio_bucket_objects "$timestamp")
+
+  download_bucket_objects
+
+  minio_stop
+
+  MIGRATION_BACKUP_DIRECTORY=$(mktemp -d -t minio-data-backup-"$timestamp"-XXXXXXXXXX)
+  echo "NOTE: Copying the old minio data to $MIGRATION_BACKUP_DIRECTORY"
+  echo "NOTE: Depending on the size of your on-prem bldr, this could take quite some time"
+  cp -r /hab/svc/builder-minio/data "$MIGRATION_BACKUP_DIRECTORY"
+  GLOBIGNORE=".:.."
+  for x in /hab/svc/builder-minio/data/*; do
+    rm -rf "$x"
+  done
+
+  export MINIO_ROOT_USER="{{cfg.env.MINIO_ACCESS_KEY}}"
+  export MINIO_ROOT_PASSWORD="{{cfg.env.MINIO_SECRET_KEY}}"
+  new_minio_stdout=$(mktemp -t "minio-new-stdout-$timestamp-XXXXXXXXXX")
+  # shellcheck disable=SC2024
+  hab pkg exec core/minio minio -- server \
+    /hab/svc/builder-minio/data \
+    --config-dir "{{pkg.svc_config_path}}" &>"$new_minio_stdout" &
+
+  if ! minio_health_live_check; then
+    echo "MinIO did not come back up so we cannot upload the habitat artifacts into the new MinIO"
+    echo "EXITING"
+    exit 1
+  fi
+
+  upload_bucket_objects
+
+  minio_new_summary=$(summarize_new_minio_bucket_objects "$timestamp")
+
+  if diff <(tail -n 2 "$minio_old_summary") <(tail -n 2 "$minio_new_summary"); then
+    echo "Migration was successful"
+    echo "A copy of the original minio backend has been left in $MIGRATION_BACKUP_DIRECTORY and should be removed manually"
+    rm -rf "$WAYPOINT"
+    echo "Old Minio bucket summary: $minio_old_summary"
+    echo "New Minio bucket summary: $minio_new_summary"
+  else
+    echo "Migration FAILED"
+    echo "Comparison of old and new MinIO bucket summaries failed"
+    echo "Leaving the old data in place for further investigation"
+    echo "Old Minio bucket summary: $minio_old_summary"
+    echo "New Minio bucket summary: $minio_new_summary"
+    echo "The migration waypoint is $WAYPOINT"
+    echo "A copy of the original minio backend data is also in $MIGRATION_BACKUP_DIRECTORY"
+  fi
+
+  minio_stop
+
+  echo "END MinIO migration"
+fi

components/builder-minio/habitat/plan.sh

@@ -1,10 +1,10 @@
-# shellcheck shell=bash
+# shellcheck shell=bash disable=SC2034
 
 pkg_name=builder-minio
 pkg_origin=habitat
 pkg_maintainer="The Habitat Maintainers <[email protected]>"
 pkg_license=('Apache-2.0')
-pkg_deps=(core/minio core/cacerts core/openssl core/aws-cli core/bash)
+pkg_deps=(core/aws-cli core/bash core/cacerts core/curl core/jq-static core/minio core/openssl)
 pkg_build_deps=(core/git)
 
 pkg_exports=(
@@ -17,7 +17,8 @@ pkg_exports=(
 pkg_version() {
   # TED: After migrating the builder repo we needed to add to
   # the rev-count to keep version sorting working
-  echo "$(($(git rev-list HEAD --count) + 5000))"
+#   echo "$(($(git rev-list HEAD --count) + 5000))"
+  echo "$(($(git rev-list HEAD --count) + 5001))"
 }
 
 do_before() {