|
| 1 | +// Copyright 2025 The Cockroach Authors. |
| 2 | +// |
| 3 | +// Use of this software is governed by the CockroachDB Software License |
| 4 | +// included in the /LICENSE file. |
| 5 | + |
| 6 | +package cli |
| 7 | + |
| 8 | +import ( |
| 9 | + "compress/gzip" |
| 10 | + "encoding/csv" |
| 11 | + "encoding/gob" |
| 12 | + "fmt" |
| 13 | + "io" |
| 14 | + "net/http" |
| 15 | + "os" |
| 16 | + "strconv" |
| 17 | + "strings" |
| 18 | + "testing" |
| 19 | + "time" |
| 20 | + |
| 21 | + "github.com/cockroachdb/cockroach/pkg/roachpb" |
| 22 | + "github.com/cockroachdb/cockroach/pkg/testutils" |
| 23 | + "github.com/cockroachdb/cockroach/pkg/ts" |
| 24 | + "github.com/cockroachdb/cockroach/pkg/ts/tspb" |
| 25 | + "github.com/cockroachdb/cockroach/pkg/util/leaktest" |
| 26 | + "github.com/cockroachdb/cockroach/pkg/util/log" |
| 27 | + "github.com/cockroachdb/datadriven" |
| 28 | + "github.com/stretchr/testify/require" |
| 29 | +) |
| 30 | + |
| 31 | +func TestTSDumpUploadE2E(t *testing.T) { |
| 32 | + defer leaktest.AfterTest(t)() |
| 33 | + defer log.Scope(t).Close(t) |
| 34 | + defer testutils.TestingHook(&getCurrentTime, func() time.Time { |
| 35 | + return time.Date(2024, 11, 14, 0, 0, 0, 0, time.UTC) |
| 36 | + })() |
| 37 | + |
| 38 | + datadriven.RunTest(t, "testdata/tsdump_upload_e2e", func(t *testing.T, d *datadriven.TestData) string { |
| 39 | + var buf strings.Builder |
| 40 | + defer mockDoDDRequest(t, &buf)() |
| 41 | + |
| 42 | + c := NewCLITest(TestCLIParams{}) |
| 43 | + defer c.Cleanup() |
| 44 | + |
| 45 | + switch d.Cmd { |
| 46 | + case "upload-datadog": |
| 47 | + dumpFilePath := generateMockTSDumpFromCSV(t, d.Input) |
| 48 | + |
| 49 | + var clusterLabel, apiKey string |
| 50 | + if d.HasArg("cluster-label") { |
| 51 | + d.ScanArgs(t, "cluster-label", &clusterLabel) |
| 52 | + } else { |
| 53 | + clusterLabel = "test-cluster" |
| 54 | + } |
| 55 | + if d.HasArg("api-key") { |
| 56 | + d.ScanArgs(t, "api-key", &apiKey) |
| 57 | + } else { |
| 58 | + apiKey = "dd-api-key" |
| 59 | + } |
| 60 | + |
| 61 | + // Run the command |
| 62 | + stdout, err := c.RunWithCapture(fmt.Sprintf( |
| 63 | + `debug tsdump --format=datadog --dd-api-key="%s" --cluster-label="%s" %s`, |
| 64 | + apiKey, clusterLabel, dumpFilePath, |
| 65 | + )) |
| 66 | + t.Log(stdout) |
| 67 | + require.NoError(t, err) |
| 68 | + return strings.TrimSpace(buf.String()) |
| 69 | + |
| 70 | + default: |
| 71 | + t.Fatalf("unknown command: %s", d.Cmd) |
| 72 | + return "" |
| 73 | + } |
| 74 | + }) |
| 75 | +} |
| 76 | + |
| 77 | +func mockDoDDRequest(t *testing.T, w io.Writer) func() { |
| 78 | + t.Helper() |
| 79 | + |
| 80 | + return testutils.TestingHook(&doDDRequest, func(req *http.Request) error { |
| 81 | + defer req.Body.Close() |
| 82 | + |
| 83 | + reader, err := gzip.NewReader(req.Body) |
| 84 | + require.NoError(t, err) |
| 85 | + |
| 86 | + raw, err := io.ReadAll(reader) |
| 87 | + require.NoError(t, err) |
| 88 | + |
| 89 | + fmt.Fprintln(w, string(raw)) |
| 90 | + return nil |
| 91 | + }) |
| 92 | +} |
| 93 | + |
| 94 | +// generateMockTSDumpFromCSV creates a mock tsdump file from CSV input string. |
| 95 | +// CSV format: metric_name,timestamp,source,value |
| 96 | +// Example: cr.node.admission.admitted.elastic-cpu,2025-05-26T08:32:00Z,1,1 |
| 97 | +// NOTE: this is the same format generated by the `cockroach tsdump` command |
| 98 | +// when --format=csv is used. |
| 99 | +func generateMockTSDumpFromCSV(t *testing.T, csvInput string) string { |
| 100 | + t.Helper() |
| 101 | + |
| 102 | + // Parse CSV data from input string |
| 103 | + reader := csv.NewReader(strings.NewReader(csvInput)) |
| 104 | + csvData, err := reader.ReadAll() |
| 105 | + require.NoError(t, err) |
| 106 | + require.Greater(t, len(csvData), 0, "CSV input must have at least one data row") |
| 107 | + |
| 108 | + // Create temporary file |
| 109 | + tmpFile, err := os.CreateTemp("", "mock_tsdump_*.gob") |
| 110 | + require.NoError(t, err) |
| 111 | + defer tmpFile.Close() |
| 112 | + |
| 113 | + // Create gob encoder |
| 114 | + encoder := gob.NewEncoder(tmpFile) |
| 115 | + |
| 116 | + // Process each row (no header expected) |
| 117 | + for i, row := range csvData { |
| 118 | + require.Len(t, row, 4, "CSV row %d must have 4 columns: metric_name,timestamp,source,value", i+1) |
| 119 | + |
| 120 | + metricName := row[0] |
| 121 | + timestampStr := row[1] |
| 122 | + source := row[2] |
| 123 | + valueStr := row[3] |
| 124 | + |
| 125 | + // Parse timestamp (RFC3339 format) |
| 126 | + timestamp, err := time.Parse(time.RFC3339, timestampStr) |
| 127 | + require.NoError(t, err, "invalid timestamp format in row %d: %s (expected RFC3339)", i+1, timestampStr) |
| 128 | + timestampNanos := timestamp.UnixNano() |
| 129 | + |
| 130 | + // Parse value |
| 131 | + value, err := strconv.ParseFloat(valueStr, 64) |
| 132 | + require.NoError(t, err, "invalid value in row %d: %s", i+1, valueStr) |
| 133 | + |
| 134 | + // Create KeyValue entry for this data point |
| 135 | + kv, err := createMockTimeSeriesKV(metricName, source, timestampNanos, value) |
| 136 | + require.NoError(t, err) |
| 137 | + |
| 138 | + // Encode to gob format |
| 139 | + err = encoder.Encode(kv) |
| 140 | + require.NoError(t, err) |
| 141 | + } |
| 142 | + |
| 143 | + t.Cleanup(func() { |
| 144 | + require.NoError(t, os.Remove(tmpFile.Name()), "failed to remove temporary file") |
| 145 | + }) |
| 146 | + return tmpFile.Name() |
| 147 | +} |
| 148 | + |
| 149 | +// createMockTimeSeriesKV creates a roachpb.KeyValue entry containing time series data |
| 150 | +func createMockTimeSeriesKV( |
| 151 | + name, source string, timestamp int64, value float64, |
| 152 | +) (roachpb.KeyValue, error) { |
| 153 | + // Create TimeSeriesData |
| 154 | + tsData := tspb.TimeSeriesData{ |
| 155 | + Name: name, |
| 156 | + Source: source, |
| 157 | + Datapoints: []tspb.TimeSeriesDatapoint{ |
| 158 | + {TimestampNanos: timestamp, Value: value}, |
| 159 | + }, |
| 160 | + } |
| 161 | + |
| 162 | + // Convert to internal format using 10s resolution |
| 163 | + resolution := ts.Resolution10s |
| 164 | + idatas, err := tsData.ToInternal( |
| 165 | + resolution.SlabDuration(), // 1 hour (3600 * 10^9 ns) |
| 166 | + resolution.SampleDuration(), // 10 seconds (10 * 10^9 ns) |
| 167 | + true, // columnar format |
| 168 | + ) |
| 169 | + if err != nil { |
| 170 | + return roachpb.KeyValue{}, err |
| 171 | + } |
| 172 | + |
| 173 | + // Should only be one internal data entry for a single datapoint |
| 174 | + if len(idatas) != 1 { |
| 175 | + return roachpb.KeyValue{}, fmt.Errorf("expected 1 internal data entry, got %d", len(idatas)) |
| 176 | + } |
| 177 | + |
| 178 | + idata := idatas[0] |
| 179 | + |
| 180 | + // Create the key |
| 181 | + key := ts.MakeDataKey(name, source, resolution, idata.StartTimestampNanos) |
| 182 | + |
| 183 | + // Create the value (protobuf-encoded internal data) |
| 184 | + var roachValue roachpb.Value |
| 185 | + if err := roachValue.SetProto(&idata); err != nil { |
| 186 | + return roachpb.KeyValue{}, err |
| 187 | + } |
| 188 | + |
| 189 | + return roachpb.KeyValue{Key: key, Value: roachValue}, nil |
| 190 | +} |
0 commit comments