Create a CLI for uploading and downloading blocks

.promu.yml

@@ -18,6 +18,8 @@ build:
           path: ./tools/scaler
         - name: tools/load-generator
           path: ./tools/load-generator
+        - name: tools/block-sync
+          path: ./tools/block-sync  
     flags: -a -tags netgo
 crossbuild:
     platforms:

go.mod

@@ -23,6 +23,50 @@ require (
 	sigs.k8s.io/kind v0.24.0
 )
 
+require (
+	cloud.google.com/go v0.115.1 // indirect
+	cloud.google.com/go/iam v1.2.0 // indirect
+	cloud.google.com/go/storage v1.43.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
+	github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible // indirect
+	github.com/aws/aws-sdk-go-v2 v1.16.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.15.1 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect
+	github.com/aws/smithy-go v1.11.1 // indirect
+	github.com/baidubce/bce-sdk-go v0.9.111 // indirect
+	github.com/clbanning/mxj v1.8.4 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
+	github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v7 v7.0.72 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mozillazg/go-httpheader v0.2.1 // indirect
+	github.com/ncw/swift v1.0.53 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.41.1 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/rs/xid v1.5.0 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
+	github.com/tencentyun/cos-go-sdk-v5 v0.7.40 // indirect
+	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+)
+
 require (
 	cloud.google.com/go/auth v0.9.5 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
@@ -42,7 +86,7 @@ require (
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
-	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-kit/log v0.2.1
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -102,6 +146,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/thanos-io/objstore v0.0.0-20240913165201-fd105025a2e5
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect

tools/block-sync/Dockerfile

@@ -0,0 +1,16 @@
+FROM alpine:latest
+LABEL maintainer="The Prometheus Authors <[email protected]>"
+
+RUN apk --no-cache add libc6-compat
+
+COPY ./block-sync /bin/block-sync
+
+# Copy the download.sh script into the container's /scripts directory
+COPY ./download.sh /scripts/download.sh
+
+# Ensure that the download.sh script is executable
+RUN chmod +x /scripts/download.sh
+
+RUN chmod +x /bin/block-sync
+
+ENTRYPOINT [ "/scripts/download.sh" ]

tools/block-sync/README.md

@@ -0,0 +1,53 @@
+
+# block-sync - TSDB Data Synchronization Tool
+
+
+The `block-sync` command is a CLI tool designed to synchronize TSDB data with an object storage system. 
+
+## Table of Contents
+
+1. [Upload](#upload)
+2. [Download](#download)
+
+## Command Flags
+
+- ``` -h , --help```:Displays context-sensitive help 
+- ``` - tsdb-path```: Path for The TSDB data in prometheus
+- ```- objstore.config-file```: Path for The Config file
+- ```- Key```: Path for the Key where to store block data , i.e a Directory. 
+
+## Upload
+
+The `upload` command allows you to upload TSDB data from a specified path to an object storage bucket. This command is essential for backing up your TSDB data or migrating it to an object storage solution for future use.
+
+### Usage
+
+```bash
+./block-sync upload --tsdb-path=<path-to-tsdb> --objstore.config-file=<path-to-config> --key=<object-key>
+
+
+```
+## Download
+
+The `download` command allows you to retrieve TSDB data from an object storage bucket to a specified local path. This command is essential for restoring your TSDB data or retrieving it for local analysis and processing.
+
+### Usage
+
+```bash
+./block-sync download --tsdb-path=<path-to-tsdb> --objstore.config-file=<path-to-config> --key=<object-key>
+```
+## Config File
+
+The configuration file is essential for connecting to your object storage solution. Below are basic templates for different object storage systems.
+
+```yaml
+type: s3, GCS , AZURE , etc.
+config:
+  bucket: your-bucket-name
+  endpoint: https://your-endpoint
+  access_key: your-access-key
+  secret_key: your-secret-key
+  insecure: false  # Set to true if using HTTP instead of HTTPS
+```
+You can customize the config file ,  follow this link [Storage.md](https://thanos.io/tip/thanos/storage.md/)
+

tools/block-sync/download.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+KEY_FILE="/key/key.yml"
+
+if [[ -f "$KEY_FILE" ]]; then
+    echo "Found key.yml, proceeding with download..."
+    /bin/block-sync download --tsdb-path=/data --objstore.config-file=/config/object-config.yml --key=$KEY_FILE
+else
+    echo "key.yml not found, skipping download."
+    exit 0
+fi

tools/block-sync/main.go

@@ -0,0 +1,103 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+)
+
+func main() {
+	var (
+		tsdbPath     string
+		objectConfig string
+		objectKey    string
+	)
+	uploadCmd := flag.NewFlagSet("upload", flag.ExitOnError)
+	downloadCmd := flag.NewFlagSet("download", flag.ExitOnError)
+
+	uploadCmd.StringVar(&tsdbPath, "tsdb-path", "", "Uploading data to objstore")
+	uploadCmd.StringVar(&objectConfig, "objstore.config-file", "", "Path for The Config file")
+	uploadCmd.StringVar(&objectKey, "key", "", "Path for the Key where to store block data")
+
+	downloadCmd.StringVar(&tsdbPath, "tsdb-path", "", "Downloading data to objstore")
+	downloadCmd.StringVar(&objectConfig, "objstore.config-file", "", "Path for The Config file")
+	downloadCmd.StringVar(&objectKey, "key", "", "Path from the Key where to download the block data")
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+		fmt.Println("  upload     Uploads data to the object store")
+		fmt.Println("  download   Downloads data from the object store")
+		fmt.Println("Flags:")
+		fmt.Println("  --tsdb-path               Path to TSDB data")
+		fmt.Println("  --objstore.config-file    Path to the object store config file")
+		fmt.Println("  --key                     Key path for storing or downloading data")
+		fmt.Println()
+		fmt.Println("Use 'block-sync [command] --help' for more information about a command.")
+	}
+
+	if len(os.Args) < 2 {
+		logger.Error("Expected 'upload' or 'download' subcommands")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	switch os.Args[1] {
+	case "upload":
+		if err := uploadCmd.Parse(os.Args[2:]); err != nil {
+			fmt.Println("Error parsing upload command:", err)
+			os.Exit(1)
+		}
+	case "download":
+		if err := downloadCmd.Parse(os.Args[2:]); err != nil {
+			fmt.Println("Error parsing download command:", err)
+			os.Exit(1)
+		}
+	default:
+		logger.Error("Expected 'upload' or 'download' subcommands")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	if tsdbPath == "" || objectConfig == "" || objectKey == "" {
+		fmt.Println("error: all flags --tsdb-path, --objstore.config-file, and --key are required.")
+		os.Exit(1)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	store, err := newStore(tsdbPath, objectConfig, objectKey, logger)
+	if err != nil {
+		logger.Error("Failed to create store", "error", err)
+		os.Exit(1)
+	}
+
+	switch os.Args[1] {
+	case "upload":
+		err = store.upload(ctx)
+		if err != nil {
+			logger.Error("Failed to upload data", "Error", err)
+			os.Exit(1)
+		}
+	case "download":
+		err = store.download(ctx)
+		if err != nil {
+			logger.Error("Failed to download data", "error", err)
+			os.Exit(1)
+		}
+	}
+}

tools/block-sync/upload_download.go

@@ -0,0 +1,110 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+
+	"github.com/go-kit/log"
+	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/client"
+)
+
+type Store struct {
+	bucket       objstore.Bucket
+	tsdbpath     string
+	objectkey    string
+	objectconfig string
+	bucketlogger *slog.Logger
+}
+
+func newStore(tsdbPath, objectConfig, objectKey string, logger *slog.Logger) (*Store, error) {
+	configBytes, err := os.ReadFile(objectConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config file: %w", err)
+	}
+
+	bucket, err := client.NewBucket(log.NewNopLogger(), configBytes, "block-sync")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create bucket existence:%w", err)
+	}
+	key, err := os.ReadFile(objectKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read objectKey file: %w", err)
+	}
+
+	content := strings.TrimSpace(string(key))
+	lines := strings.Split(content, "\n")
+	var value string
+
+	// Loop through each line to find the key
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "key:") {
+			// Extract the value after "key:"
+			value = strings.TrimSpace(strings.TrimPrefix(line, "key:"))
+			break
+		}
+	}
+
+	if value == "" {
+		return nil, fmt.Errorf("expected 'key:' prefix not found")
+	}
+
+	return &Store{
+		bucket:       bucket,
+		tsdbpath:     tsdbPath,
+		objectkey:    value,
+		objectconfig: objectConfig,
+		bucketlogger: logger,
+	}, nil
+}
+
+func (c *Store) upload(ctx context.Context) error {
+	exists, err := c.bucket.Exists(ctx, c.objectkey)
+	if err != nil {
+		return fmt.Errorf("failed to check new bucket:%w", err)
+	}
+	c.bucketlogger.Info("Bucket checked  Successfully", "Bucket name", exists)
+
+	err = objstore.UploadDir(ctx, log.NewNopLogger(), c.bucket, c.tsdbpath, c.objectkey)
+	if err != nil {
+		c.bucketlogger.Error("Failed to upload directory", "path", c.tsdbpath, "error", err)
+		return fmt.Errorf("failed to upload directory from path %s to bucket: %w", c.tsdbpath, err)
+	}
+
+	c.bucketlogger.Info("Successfully uploaded directory", "path", c.tsdbpath, "bucket", c.bucket.Name())
+	return nil
+}
+
+func (c *Store) download(ctx context.Context) error {
+	exists, err := c.bucket.Exists(ctx, c.objectkey)
+	if err != nil {
+		return fmt.Errorf("failed to check new bucket:%w", err)
+	}
+	c.bucketlogger.Info("Bucket checked  Successfully", "Bucket name", exists)
+
+	err = objstore.DownloadDir(ctx, log.NewNopLogger(), c.bucket, "dir/", c.objectkey, c.tsdbpath)
+	if err != nil {
+		c.bucketlogger.Error("Failed to download directory", "path", c.tsdbpath, "error", err)
+		return fmt.Errorf("failed to download directory from path %s to bucket: %w", c.tsdbpath, err)
+	}
+
+	c.bucketlogger.Info("Successfully downloaded directory", "path", c.tsdbpath, "bucket", c.bucket.Name())
+	return nil
+}