From 97d7277c6bfaf869bfb469f3a666751bdda353ee Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Thu, 21 Nov 2024 21:05:03 +0530 Subject: [PATCH 1/3] a code for exiting container if secretes is empty. Signed-off-by: Kushal Shukla --- tools/block-sync/upload_download.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/block-sync/upload_download.go b/tools/block-sync/upload_download.go index 3bd3dc213..363c477b0 100644 --- a/tools/block-sync/upload_download.go +++ b/tools/block-sync/upload_download.go @@ -38,6 +38,10 @@ func newStore(tsdbPath, objectConfig, objectKey string, logger *slog.Logger) (*S if err != nil { return nil, fmt.Errorf("failed to read config file: %w", err) } + if len(configBytes) == 0 { + fmt.Println("Config file is empty, exiting container.") + os.Exit(0) + } bucket, err := client.NewBucket(log.NewNopLogger(), configBytes, "block-sync") if err != nil { From 1f45cce50d921762f450012519edb180c361aa4d Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Thu, 21 Nov 2024 21:09:30 +0530 Subject: [PATCH 2/3] removed red line from bottom. Signed-off-by: Kushal Shukla --- tools/block-sync/Dockerfile | 2 +- tools/block-sync/download.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/block-sync/Dockerfile b/tools/block-sync/Dockerfile index 55e7cc42e..5b0292de6 100644 --- a/tools/block-sync/Dockerfile +++ b/tools/block-sync/Dockerfile @@ -13,4 +13,4 @@ RUN chmod +x /scripts/download.sh RUN chmod +x /bin/block-sync -ENTRYPOINT [ "/scripts/download.sh" ] \ No newline at end of file +ENTRYPOINT [ "/scripts/download.sh" ] diff --git a/tools/block-sync/download.sh b/tools/block-sync/download.sh index 024a69080..790d3af4c 100755 --- a/tools/block-sync/download.sh +++ b/tools/block-sync/download.sh @@ -8,4 +8,4 @@ if [[ -f "$KEY_FILE" ]]; then else echo "key.yml not found, skipping download." exit 0 -fi \ No newline at end of file +fi From 6350c7631af08a0ea1d8d5eb4b07bc1d375de441 Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Fri, 22 Nov 2024 14:23:23 +0530 Subject: [PATCH 3/3] 1. Renamed key.yml to object-config.yml file. 2. Replaced key with path and other parts of the file. Signed-off-by: Kushal Shukla --- tools/block-sync/README.md | 10 +++++----- tools/block-sync/download.sh | 10 +++++----- tools/block-sync/main.go | 15 +++++++-------- tools/block-sync/upload_download.go | 30 ++++++++++++++--------------- 4 files changed, 31 insertions(+), 34 deletions(-) diff --git a/tools/block-sync/README.md b/tools/block-sync/README.md index 51c3e2a57..5d88f53db 100644 --- a/tools/block-sync/README.md +++ b/tools/block-sync/README.md @@ -1,7 +1,5 @@ - # block-sync - TSDB Data Synchronization Tool - The `block-sync` command is a CLI tool designed to synchronize TSDB data with an object storage system. ## Table of Contents @@ -14,7 +12,9 @@ The `block-sync` command is a CLI tool designed to synchronize TSDB data with an - ``` -h , --help```:Displays context-sensitive help - ``` - tsdb-path```: Path for The TSDB data in prometheus - ```- objstore.config-file```: Path for The Config file -- ```- Key```: Path for the Key where to store block data , i.e a Directory. +- ```- path```: Path within the objectstorage where to store block data , i.e a Directory. +## Note +> Use the `path` flag with a file specifying the path: *your directory name*. ## Upload @@ -23,7 +23,7 @@ The `upload` command allows you to upload TSDB data from a specified path to an ### Usage ```bash -./block-sync upload --tsdb-path= --objstore.config-file= --key= +./block-sync upload --tsdb-path= --objstore.config-file= --path= ``` @@ -34,7 +34,7 @@ The `download` command allows you to retrieve TSDB data from an object storage b ### Usage ```bash -./block-sync download --tsdb-path= --objstore.config-file= --key= +./block-sync download --tsdb-path= --objstore.config-file= --path= ``` ## Config File diff --git a/tools/block-sync/download.sh b/tools/block-sync/download.sh index 790d3af4c..5f8df0cea 100755 --- a/tools/block-sync/download.sh +++ b/tools/block-sync/download.sh @@ -1,11 +1,11 @@ #!/bin/sh -KEY_FILE="/key/key.yml" +PATH_FILE="/key/bucket-config.yml" -if [[ -f "$KEY_FILE" ]]; then - echo "Found key.yml, proceeding with download..." - /bin/block-sync download --tsdb-path=/data --objstore.config-file=/config/object-config.yml --key=$KEY_FILE +if [[ -f "$PATH_FILE" ]]; then + echo "Found bucket-config.yml, proceeding with download..." + /bin/block-sync download --tsdb-path=/data --objstore.config-file=/config/object-config.yml --path=$PATH_FILE else - echo "key.yml not found, skipping download." + echo "bucket-config.yml not found, skipping download." exit 0 fi diff --git a/tools/block-sync/main.go b/tools/block-sync/main.go index b78deae8a..2818bcd5b 100644 --- a/tools/block-sync/main.go +++ b/tools/block-sync/main.go @@ -25,18 +25,18 @@ func main() { var ( tsdbPath string objectConfig string - objectKey string + objectPath string ) uploadCmd := flag.NewFlagSet("upload", flag.ExitOnError) downloadCmd := flag.NewFlagSet("download", flag.ExitOnError) uploadCmd.StringVar(&tsdbPath, "tsdb-path", "", "Uploading data to objstore") uploadCmd.StringVar(&objectConfig, "objstore.config-file", "", "Path for The Config file") - uploadCmd.StringVar(&objectKey, "key", "", "Path for the Key where to store block data") + uploadCmd.StringVar(&objectPath, "path", "", "Path within the objectstorage") downloadCmd.StringVar(&tsdbPath, "tsdb-path", "", "Downloading data to objstore") downloadCmd.StringVar(&objectConfig, "objstore.config-file", "", "Path for The Config file") - downloadCmd.StringVar(&objectKey, "key", "", "Path from the Key where to download the block data") + downloadCmd.StringVar(&objectPath, "path", "", "Path within the objectstorage") logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) flag.Usage = func() { @@ -46,9 +46,8 @@ func main() { fmt.Println("Flags:") fmt.Println(" --tsdb-path Path to TSDB data") fmt.Println(" --objstore.config-file Path to the object store config file") - fmt.Println(" --key Key path for storing or downloading data") + fmt.Println(" --path Directory in objectstorage for uploading or downloading data") fmt.Println() - fmt.Println("Use 'block-sync [command] --help' for more information about a command.") } if len(os.Args) < 2 { @@ -74,13 +73,13 @@ func main() { os.Exit(1) } - if tsdbPath == "" || objectConfig == "" || objectKey == "" { - fmt.Println("error: all flags --tsdb-path, --objstore.config-file, and --key are required.") + if tsdbPath == "" || objectConfig == "" || objectPath == "" { + fmt.Println("error: all flags --tsdb-path, --objstore.config-file, and --path are required.") os.Exit(1) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - store, err := newStore(tsdbPath, objectConfig, objectKey, logger) + store, err := newStore(tsdbPath, objectConfig, objectPath, logger) if err != nil { logger.Error("Failed to create store", "error", err) os.Exit(1) diff --git a/tools/block-sync/upload_download.go b/tools/block-sync/upload_download.go index 363c477b0..d61d4d55e 100644 --- a/tools/block-sync/upload_download.go +++ b/tools/block-sync/upload_download.go @@ -28,12 +28,12 @@ import ( type Store struct { bucket objstore.Bucket tsdbpath string - objectkey string + objectpath string objectconfig string bucketlogger *slog.Logger } -func newStore(tsdbPath, objectConfig, objectKey string, logger *slog.Logger) (*Store, error) { +func newStore(tsdbPath, objectConfig, objectPath string, logger *slog.Logger) (*Store, error) { configBytes, err := os.ReadFile(objectConfig) if err != nil { return nil, fmt.Errorf("failed to read config file: %w", err) @@ -47,46 +47,44 @@ func newStore(tsdbPath, objectConfig, objectKey string, logger *slog.Logger) (*S if err != nil { return nil, fmt.Errorf("failed to create bucket existence:%w", err) } - key, err := os.ReadFile(objectKey) + path, err := os.ReadFile(objectPath) if err != nil { return nil, fmt.Errorf("failed to read objectKey file: %w", err) } - content := strings.TrimSpace(string(key)) + content := strings.TrimSpace(string(path)) lines := strings.Split(content, "\n") - var value string + var directory string - // Loop through each line to find the key for _, line := range lines { line = strings.TrimSpace(line) - if strings.HasPrefix(line, "key:") { - // Extract the value after "key:" - value = strings.TrimSpace(strings.TrimPrefix(line, "key:")) + if strings.HasPrefix(line, "path:") { + directory = strings.TrimSpace(strings.TrimPrefix(line, "path:")) break } } - if value == "" { - return nil, fmt.Errorf("expected 'key:' prefix not found") + if directory == "" { + return nil, fmt.Errorf("expected 'path:' prefix not found") } return &Store{ bucket: bucket, tsdbpath: tsdbPath, - objectkey: value, + objectpath: directory, objectconfig: objectConfig, bucketlogger: logger, }, nil } func (c *Store) upload(ctx context.Context) error { - exists, err := c.bucket.Exists(ctx, c.objectkey) + exists, err := c.bucket.Exists(ctx, c.objectpath) if err != nil { return fmt.Errorf("failed to check new bucket:%w", err) } c.bucketlogger.Info("Bucket checked Successfully", "Bucket name", exists) - err = objstore.UploadDir(ctx, log.NewNopLogger(), c.bucket, c.tsdbpath, c.objectkey) + err = objstore.UploadDir(ctx, log.NewNopLogger(), c.bucket, c.tsdbpath, c.objectpath) if err != nil { c.bucketlogger.Error("Failed to upload directory", "path", c.tsdbpath, "error", err) return fmt.Errorf("failed to upload directory from path %s to bucket: %w", c.tsdbpath, err) @@ -97,13 +95,13 @@ func (c *Store) upload(ctx context.Context) error { } func (c *Store) download(ctx context.Context) error { - exists, err := c.bucket.Exists(ctx, c.objectkey) + exists, err := c.bucket.Exists(ctx, c.objectpath) if err != nil { return fmt.Errorf("failed to check new bucket:%w", err) } c.bucketlogger.Info("Bucket checked Successfully", "Bucket name", exists) - err = objstore.DownloadDir(ctx, log.NewNopLogger(), c.bucket, "dir/", c.objectkey, c.tsdbpath) + err = objstore.DownloadDir(ctx, log.NewNopLogger(), c.bucket, "dir/", c.objectpath, c.tsdbpath) if err != nil { c.bucketlogger.Error("Failed to download directory", "path", c.tsdbpath, "error", err) return fmt.Errorf("failed to download directory from path %s to bucket: %w", c.tsdbpath, err)