cli: upload misc. files to datadog from debug.zip

This patch extends the existing debug.zip upload functionality to datadog. This patch uploads cluster wide generated misc. files to datadog. Epic: CC-28996 Part of: CC-28884 Release note: None
cockroachdb · Jan 22, 2025 · dd17180 · dd17180
1 parent ad49a4a
commit dd17180
Show file tree

Hide file tree

Showing 7 changed files with 164 additions and 1 deletion.
diff --git a/pkg/cli/testdata/table_dumps/events.json b/pkg/cli/testdata/table_dumps/events.json
@@ -0,0 +1,11 @@
+{
+  "events": [
+    {
+      "timestamp": "2025-01-21T05:17:42.857695Z",
+      "event_type": "node_restart",
+      "reporting_id": 1,
+      "info": "{\"EventType\":\"node_restart\",\"LastUp\":1737426359068817000,\"NodeID\":1,\"StartedAt\":1737436662756679000,\"Timestamp\":1737436662857695000}",
+      "unique_id": "Kmuncp6KTf+KWiuO3I4aww=="
+    }
+  ]
+}
diff --git a/pkg/cli/testdata/table_dumps/rangelog.json b/pkg/cli/testdata/table_dumps/rangelog.json
@@ -0,0 +1,52 @@
+{
+  "events": [
+    {
+      "event": {
+        "timestamp": "2025-01-20T11:52:24.424851Z",
+        "range_id": 51,
+        "store_id": 1,
+        "event_type": 3,
+        "other_range_id": 57,
+        "info": {
+          "UpdatedDesc": {
+            "range_id": 51,
+            "start_key": "uA==",
+            "end_key": "wg==",
+            "internal_replicas": [
+              {
+                "node_id": 1,
+                "store_id": 1,
+                "replica_id": 1,
+                "type": 0
+              }
+            ],
+            "next_replica_id": 2,
+            "generation": 5,
+            "sticky_bit": {}
+          },
+          "RemovedDesc": {
+            "range_id": 57,
+            "start_key": "vw==",
+            "end_key": "wg==",
+            "internal_replicas": [
+              {
+                "node_id": 1,
+                "store_id": 1,
+                "replica_id": 1,
+                "type": 0
+              }
+            ],
+            "next_replica_id": 2,
+            "generation": 2,
+            "sticky_bit": {}
+          }
+        }
+      },
+      "pretty_info": {
+        "updated_desc": "r51:/Table/{48-58} [(n1,s1):1, next=2, gen=5]"
+      }
+    }
+  ]
+}
+
+
diff --git a/pkg/cli/testdata/table_dumps/reports/problemranges.json b/pkg/cli/testdata/table_dumps/reports/problemranges.json
@@ -0,0 +1,6 @@
+{
+  "node_id": 1,
+  "problems_by_node_id": {
+    "1": {}
+  }
+}
diff --git a/pkg/cli/testdata/table_dumps/settings.json b/pkg/cli/testdata/table_dumps/settings.json
@@ -0,0 +1,11 @@
+{
+  "key_values": {
+    "admission.disk_bandwidth_tokens.elastic.enabled": {
+      "value": "true",
+      "type": "b",
+      "description": "when true, and provisioned bandwidth for the disk corresponding to a store is configured, tokens for elastic work will be limited if disk bandwidth becomes a bottleneck",
+      "public": true,
+      "name": "admission.disk_bandwidth_tokens.elastic.enabled"
+    }
+  }
+}
diff --git a/pkg/cli/testdata/upload/misc b/pkg/cli/testdata/upload/misc
@@ -0,0 +1,21 @@
+upload-misc
+{
+  "nodes": {
+    "1": {}
+  }
+}
+----
+Logs API Hook: https://http-intake.logs.us5.datadoghq.com/api/v2/logs
+Logs API Hook: https://http-intake.logs.us5.datadoghq.com/api/v2/logs
+Logs API Hook: https://http-intake.logs.us5.datadoghq.com/api/v2/logs
+Logs API Hook: https://http-intake.logs.us5.datadoghq.com/api/v2/logs
+Upload ID: abc-20241114000000
+body: {"message":{"events":[{"event":{"timestamp":"2025-01-20T11:52:24.424851Z","range_id":51,"store_id":1,"event_type":3,"other_range_id":57,"info":{"UpdatedDesc":{"range_id":51,"start_key":"uA==","end_key":"wg==","internal_replicas":[{"node_id":1,"store_id":1,"replica_id":1,"type":0}],"next_replica_id":2,"generation":5,"sticky_bit":{}},"RemovedDesc":{"range_id":57,"start_key":"vw==","end_key":"wg==","internal_replicas":[{"node_id":1,"store_id":1,"replica_id":1,"type":0}],"next_replica_id":2,"generation":2,"sticky_bit":{}}}},"pretty_info":{"updated_desc":"r51:/Table/{48-58} [(n1,s1):1, next=2, gen=5]"}}]},"tags":["cluster:ABC","upload_id:abc-20241114000000"]}
+body: {"message":{"events":[{"timestamp":"2025-01-21T05:17:42.857695Z","event_type":"node_restart","reporting_id":1,"info":"{\"EventType\":\"node_restart\",\"LastUp\":1737426359068817000,\"NodeID\":1,\"StartedAt\":1737436662756679000,\"Timestamp\":1737436662857695000}","unique_id":"Kmuncp6KTf+KWiuO3I4aww=="}]},"tags":["cluster:ABC","upload_id:abc-20241114000000"]}
+body: {"message":{"key_values":{"admission.disk_bandwidth_tokens.elastic.enabled":{"value":"true","type":"b","description":"when true, and provisioned bandwidth for the disk corresponding to a store is configured, tokens for elastic work will be limited if disk bandwidth becomes a bottleneck","public":true,"name":"admission.disk_bandwidth_tokens.elastic.enabled"}}},"tags":["cluster:ABC","upload_id:abc-20241114000000"]}
+body: {"message":{"node_id":1,"problems_by_node_id":{"1":{}}},"tags":["cluster:ABC","upload_id:abc-20241114000000"]}
+debug zip upload debugDir --dd-api-key=dd-api-key --dd-app-key=dd-app-key --cluster=ABC --include=misc
+uploaded events.json
+uploaded rangelog.json
+uploaded reports/problemranges.json
+uploaded settings.json
diff --git a/pkg/cli/zip_upload.go b/pkg/cli/zip_upload.go
@@ -28,6 +28,7 @@ import (
 
 	"cloud.google.com/go/storage"
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/util/httputil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/log/logpb"
@@ -132,7 +133,7 @@ var debugZipUploadOpts = struct {
 // var zipArtifactTypes = []string{"profiles", "logs"}
 // TODO(arjunmahishi): Removing the profiles upload for now. It has started
 // failing for some reason. Will fix this later
-var zipArtifactTypes = []string{"logs", "tables"}
+var zipArtifactTypes = []string{"logs", "tables", "misc"}
 
 // uploadZipArtifactFuncs is a registry of handler functions for each artifact type.
 // While adding/removing functions from here, make sure to update
@@ -141,6 +142,61 @@ var uploadZipArtifactFuncs = map[string]uploadZipArtifactFunc{
 	"profiles": uploadZipProfiles,
 	"logs":     uploadZipLogs,
 	"tables":   uploadZipTables,
+	"misc":     uploadMiscFiles,
+}
+
+func uploadMiscFiles(ctx context.Context, uuid string, dirPath string) error {
+	files := []struct {
+		fileName string
+		message  any
+	}{
+		{settingsFile, &serverpb.SettingsResponse{}},
+		{eventsFile, &serverpb.EventsResponse{}},
+		{rangeLogFile, &serverpb.RangeLogResponse{}},
+		{path.Join("reports", problemRangesFile), &serverpb.ProblemRangesResponse{}},
+	}
+
+	for _, file := range files {
+		if err := parseJSONFile(dirPath, file.fileName, file.message); err != nil {
+			fmt.Fprintf(os.Stderr, "parsing failed for file: %s with error: %s\n", file.fileName, err)
+			continue
+		}
+
+		if err := uploadJSONFile(file.fileName, file.message, uuid); err != nil {
+			fmt.Fprintf(os.Stderr, "upload failed for file: %s with error: %s\n", file.fileName, err)
+		} else {
+			fmt.Fprintf(os.Stderr, "uploaded %s\n", file.fileName)
+		}
+	}
+	return nil
+}
+
+func uploadJSONFile(fileName string, message any, uuid string) error {
+
+	body, err := json.Marshal(struct {
+		Message any    `json:"message"`
+		DDTags  string `json:"ddtags"`
+	}{
+		Message: message,
+		DDTags: strings.Join(appendUserTags(
+			append([]string{}, makeDDTag(uploadIDTag, uuid),
+				makeDDTag(clusterTag, debugZipUploadOpts.clusterName),
+				makeDDTag("file_name", fileName),
+			), // system generated tags
+			debugZipUploadOpts.tags..., // user provided tags
+		), ","),
+	})
+	if err != nil {
+		return err
+	}
+
+	_, err = uploadLogsToDatadog(
+		body, debugZipUploadOpts.ddAPIKey, debugZipUploadOpts.ddSite,
+	)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // default datadog tags. Source has to be "cockroachdb" for the logs to be

diff --git a/pkg/cli/zip_upload_test.go b/pkg/cli/zip_upload_test.go
@@ -208,6 +208,8 @@ func TestUploadZipEndToEnd(t *testing.T) {
 				}
 			case "upload-tables":
 				includeFlag = "--include=tables"
+			case "upload-misc":
+				includeFlag = "--include=misc"
 			}
 
 			debugDir, cleanup := setupZipDir(t, testInput)
@@ -419,6 +421,8 @@ func setupDDLogsHook(t *testing.T, req *http.Request) ([]byte, error) {
 			}
 			fmt.Println()
 		}
+	} else {
+		fmt.Printf("body: %s\n", body.String())
 	}
 
 	return []byte("200 OK"), nil
@@ -574,6 +578,8 @@ func copyZipFiles(t *testing.T, src, dest string) {
 	paths, err := expandPatterns([]string{
 		path.Join(src, "*.txt"),
 		path.Join(src, "nodes/*/*.txt"),
+		path.Join(src, "*.json"),
+		path.Join(src, "reports/*.json"),
 	})
 	require.NoError(t, err)