Merge remote-tracking branch 'upstream/main' into 20240626_add-collec…

…tion-rules-handling

.github/workflows/test.yml

@@ -63,10 +63,10 @@ jobs:
   doc-validator:
     runs-on: "ubuntu-latest"
     container:
-      image: "grafana/doc-validator:v4.1.1"
+      image: "grafana/doc-validator:v5.1.0"
     steps:
       - name: "Checkout code"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
       - name: "Run doc-validator tool"
         run: >
           doc-validator

.gitignore

@@ -15,6 +15,7 @@ pyroscope-sync/
 data/
 data-shared/
 data-compactor/
+data-metastore/
 .DS_Store
 **/dist
 

.golangci.yml

@@ -81,6 +81,8 @@ issues:
   exclude-dirs:
     - win_eventlog$
     - pkg/og
+    - pkg/experiment
+
   # which files to skip: they will be analyzed, but issues from them
   # won't be reported. Default value is empty list, but there is
   # no need to include all autogenerated files, we confidently recognize

api/compactor/v1/compactor.proto

@@ -0,0 +1,95 @@
+syntax = "proto3";
+
+package compactor.v1;
+
+import "metastore/v1/metastore.proto";
+
+service CompactionPlanner {
+  // Used to both retrieve jobs and update the jobs status at the same time.
+  rpc PollCompactionJobs(PollCompactionJobsRequest) returns (PollCompactionJobsResponse) {}
+  // Used for admin purposes only.
+  rpc GetCompactionJobs(GetCompactionRequest) returns (GetCompactionResponse) {}
+}
+
+message PollCompactionJobsRequest {
+  // A batch of status updates for in-progress jobs from a worker.
+  repeated CompactionJobStatus job_status_updates = 1;
+  // How many new jobs a worker can be assigned to.
+  uint32 job_capacity = 2;
+}
+
+message PollCompactionJobsResponse {
+  repeated CompactionJob compaction_jobs = 1;
+}
+
+message GetCompactionRequest {}
+
+message GetCompactionResponse {
+  // A list of all compaction jobs
+  repeated CompactionJob compaction_jobs = 1;
+}
+
+// One compaction job may result in multiple output blocks.
+message CompactionJob {
+  // Unique name of the job.
+  string name = 1;
+  CompactionOptions options = 2;
+  // List of the input blocks.
+  repeated metastore.v1.BlockMeta blocks = 3;
+  CompactionJobStatus status = 4;
+  // Fencing token.
+  uint64 raft_log_index = 5;
+  // Shard the blocks belong to.
+  uint32 shard = 6;
+  // Optional, empty for compaction level 0.
+  string tenant_id = 7;
+  uint32 compaction_level = 8;
+}
+
+message CompactionOptions {
+  // Compaction planner should instruct the compactor
+  // worker how to compact the blocks:
+  //  - Limits and tenant overrides.
+  //  - Feature flags.
+
+  // How often the compaction worker should update
+  // the job status. If overdue, the job ownership
+  // is revoked.
+  uint64 status_update_interval_seconds = 1;
+}
+
+message CompactionJobStatus {
+  string job_name = 1;
+  // Status update allows the planner to keep
+  // track of the job ownership and compaction
+  // progress:
+  // - If the job status is other than IN_PROGRESS,
+  //   the ownership of the job is revoked.
+  // - FAILURE must only be sent if the failure is
+  //   persistent and the compaction can't be accomplished.
+  // - completed_job must be empty if the status is
+  //   other than SUCCESS, and vice-versa.
+  // - UNSPECIFIED must be sent if the worker rejects
+  //   or cancels the compaction job.
+  //
+  // Partial results/status is not allowed.
+  CompactionStatus status = 2;
+  CompletedJob completed_job = 3;
+  // Fencing token.
+  uint64 raft_log_index = 4;
+  // Shard the blocks belong to.
+  uint32 shard = 5;
+  // Optional, empty for compaction level 0.
+  string tenant_id = 6;
+}
+
+enum CompactionStatus {
+  COMPACTION_STATUS_UNSPECIFIED = 0;
+  COMPACTION_STATUS_IN_PROGRESS = 1;
+  COMPACTION_STATUS_SUCCESS = 2;
+  COMPACTION_STATUS_FAILURE = 3;
+}
+
+message CompletedJob {
+  repeated metastore.v1.BlockMeta blocks = 1;
+}