diff --git a/tools/object-storage/Dockerfile b/tools/blockSyncer/Dockerfile similarity index 62% rename from tools/object-storage/Dockerfile rename to tools/blockSyncer/Dockerfile index 5844c4fc9..0b18cb3a0 100644 --- a/tools/object-storage/Dockerfile +++ b/tools/blockSyncer/Dockerfile @@ -1,6 +1,6 @@ FROM quay.io/prometheus/busybox:latest LABEL maintainer="The Prometheus Authors " -COPY object-storage /bin/object-storage +COPY blockSyncer /bin/blocksyncer -ENTRYPOINT ["/bin/object-storage"] +ENTRYPOINT ["/bin/blocksyncer"] diff --git a/tools/blockSyncer/Readme.md b/tools/blockSyncer/Readme.md new file mode 100644 index 000000000..762c33dc9 --- /dev/null +++ b/tools/blockSyncer/Readme.md @@ -0,0 +1,55 @@ + +# blockSynch - TSDB Data Synchronization Tool + + +The `block-sync` command is a CLI tool designed to synchronize TSDB data with an object storage system. + +## Table of Contents + +1. [Upload](#upload) +2. [Download](#download) + +## Command Flags + +- ``` -h , --help```:Displays context-sensitive help +- ``` - tsdb-path```: Path for The TSDB data in prometheus +- ```- objstore.config-file```: Path for The Config file +- ```- Key```: Path for the Key where to store block data , i.e a Directory. + +## Upload + +The `upload` command allows you to upload TSDB data from a specified path to an object storage bucket. This command is essential for backing up your TSDB data or migrating it to an object storage solution for future use. + +### Usage + +```bash +./blockSynch upload --tsdb-path= --objstore.config-file= --key= + + +``` +## Download + +The `download` command allows you to retrieve TSDB data from an object storage bucket to a specified local path. This command is essential for restoring your TSDB data or retrieving it for local analysis and processing. + +### Usage + +```bash +./blockSynch download --tsdb-path= --objstore.config-file= --key= +``` +## Config File + +The configuration file is essential for connecting to your object storage solution. Below are basic templates for different object storage systems. + +### MinIO + +```yaml +type: s3 +config: + bucket: your-bucket-name + endpoint: https://your-minio-endpoint.com + access_key: your-access-key + secret_key: your-secret-key + insecure: false # Set to true if using HTTP instead of HTTPS +``` +For more follow this link [Storage.md](https://thanos.io/tip/thanos/storage.md/) + diff --git a/tools/blockSyncer/obj.go b/tools/blockSyncer/obj.go new file mode 100644 index 000000000..50b4d5d38 --- /dev/null +++ b/tools/blockSyncer/obj.go @@ -0,0 +1,50 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + + "gopkg.in/alecthomas/kingpin.v2" +) + +func main() { + app := kingpin.New(filepath.Base(os.Args[0]), "Tool for storing TSDB data to object storage") + app.HelpFlag.Short('h') + + var tsdbPath, objectConfig, objectKey string + var s *Store + + objstore := app.Command("block-sync", `Using an object storage to store the data`) + objstore.Flag("tsdb-path", "Path for The TSDB data in prometheus").Required().StringVar(&tsdbPath) + objstore.Flag("objstore.config-file", "Path for The Config file").Required().StringVar(&objectConfig) + objstore.Flag("key", "Path for the Key where to store block data").Required().StringVar(&objectKey) + + objstore.Action(func(c *kingpin.ParseContext) error { + s = newstore(tsdbPath, objectConfig, objectKey) + return nil + }) + + uploadCmd := objstore.Command("upload", "Uploading data") + uploadCmd.Action(func(c *kingpin.ParseContext) error { + return s.upload(c) + }) + + downloadCmd := objstore.Command("download", "Downloading data") + downloadCmd.Action(func(c *kingpin.ParseContext) error { + return s.download(c) + }) + kingpin.MustParse(app.Parse(os.Args[1:])) +} diff --git a/tools/blockSyncer/upload_download.go b/tools/blockSyncer/upload_download.go new file mode 100644 index 000000000..188d58f1f --- /dev/null +++ b/tools/blockSyncer/upload_download.go @@ -0,0 +1,90 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/go-kit/log" + "github.com/thanos-io/objstore" + "github.com/thanos-io/objstore/client" + "gopkg.in/alecthomas/kingpin.v2" +) + +type Store struct { + Bucket objstore.Bucket + TsdbPath string + ObjectKey string + ObjectConfig string +} + +func newstore(tsdbPath, objectConfig, objectKey string) *Store { + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + configBytes, err := os.ReadFile(objectConfig) + if err != nil { + logger.Error("failed to read config file", "error", err) + return nil + } + + bucket, err := client.NewBucket(log.NewNopLogger(), configBytes, "block-sync") + if err != nil { + logger.Error("Failed to create bucket existence", "error", err) + return nil + } + + exists, err := bucket.Exists(context.Background(), objectKey) + if err != nil { + logger.Error("Failed to create new bucket", "error", err) + return nil + } + logger.Info("Bucket existence check", "bucket_name", objectKey, "exists", exists) + + return &Store{ + Bucket: bucket, + TsdbPath: tsdbPath, + ObjectConfig: objectConfig, + ObjectKey: objectKey, + } +} + +func (c *Store) upload(*kingpin.ParseContext) error { + + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + + err := objstore.UploadDir(context.Background(), log.NewNopLogger(), c.Bucket, c.TsdbPath, c.ObjectKey) + if err != nil { + logger.Error("Failed to upload directory", "path", c.TsdbPath, "error", err) + return fmt.Errorf("failed to upload directory from path %s to bucket: %v", c.TsdbPath, err) + } + + logger.Info("Successfully uploaded directory", "path", c.TsdbPath, "bucket", c.Bucket.Name()) + return nil +} + +func (c *Store) download(*kingpin.ParseContext) error { + + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + + err := objstore.DownloadDir(context.Background(), log.NewNopLogger(), c.Bucket, "dir/", c.ObjectKey, c.TsdbPath) + if err != nil { + logger.Error("Failed to download directory", "path", c.TsdbPath, "error", err) + return fmt.Errorf("failed to download directory from path %s to bucket: %v", c.TsdbPath, err) + } + + logger.Info("Successfully downloaded directory", "path", c.TsdbPath, "bucket", c.Bucket.Name()) + return nil +} diff --git a/tools/object-storage/obj.go b/tools/object-storage/obj.go deleted file mode 100644 index 6edcad322..000000000 --- a/tools/object-storage/obj.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2024 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "path/filepath" - - "gopkg.in/alecthomas/kingpin.v2" -) - -func main() { - app := kingpin.New(filepath.Base(os.Args[0]), "Tool for storing TSDB data to object storage") - app.HelpFlag.Short('h') - - s := newstore() - Objstore := app.Command("block-sync", `Using Thanos to store the data`) - Objstore.Flag("path", "Path for The TSDB data in prometheus").Required().StringVar(&s.TsdbPath) - Objstore.Flag("objstore.config-file", "Path for The Config file").Required().StringVar(&s.ObjectConfig) - Objstore.Flag("key", "Path for the Key where to store block data").Required().StringVar(&s.ObjectKey) - Objstore.Command("upload", "Uploading data").Action(s.upload) - Objstore.Command("download", "Downloading data").Action(s.download) - kingpin.MustParse(app.Parse(os.Args[1:])) -} diff --git a/tools/object-storage/upload_download.go b/tools/object-storage/upload_download.go deleted file mode 100644 index 6f8744ca6..000000000 --- a/tools/object-storage/upload_download.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2024 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "os" - - "github.com/go-kit/log" - "github.com/thanos-io/objstore" - "github.com/thanos-io/objstore/client" - "gopkg.in/alecthomas/kingpin.v2" -) - -type Store struct { - TsdbPath string - ObjectKey string - ObjectConfig string -} - -func newstore() *Store { - return &Store{ - TsdbPath: "", - ObjectKey: "", - ObjectConfig: "", - } -} - -func (c *Store) upload(*kingpin.ParseContext) error { - config := c.ObjectConfig - configBytes, err := os.ReadFile(config) - if err != nil { - panic(fmt.Errorf("failed to read config file: %v", err)) - } - commitDir := c.ObjectKey - - bucket, err := client.NewBucket(log.NewNopLogger(), configBytes, "example") - if err != nil { - panic(err) - } - exists, err := bucket.Exists(context.Background(), "example") - if err != nil { - panic(err) - } - fmt.Println(exists) - - err = objstore.UploadDir(context.Background(), log.NewNopLogger(), bucket, c.TsdbPath, commitDir) - if err != nil { - panic(err) - } - return nil -} - -func (c *Store) download(*kingpin.ParseContext) error { - config := c.ObjectConfig - configBytes, err := os.ReadFile(config) - if err != nil { - panic(fmt.Errorf("failed to read config file: %v", err)) - } - commitDir := c.ObjectKey - bucket, err := client.NewBucket(log.NewNopLogger(), configBytes, "example") - if err != nil { - panic(err) - } - exists, err := bucket.Exists(context.Background(), "example") - if err != nil { - return fmt.Errorf("failed to create bucket: %v", err) - } - fmt.Println(exists) - - err = objstore.DownloadDir(context.Background(), log.NewNopLogger(), bucket, "dir/", commitDir, c.TsdbPath) - if err != nil { - return fmt.Errorf("failed to download directory: %v", err) - } - return nil -}