From ab5319476173d06b30430d2c40291c1dace26492 Mon Sep 17 00:00:00 2001 From: Adnan Gulegulzar Date: Fri, 7 Jun 2024 19:59:44 -0400 Subject: [PATCH 1/4] initialize CloudMonitoring --- plugin/gcp/metrics.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 plugin/gcp/metrics.go diff --git a/plugin/gcp/metrics.go b/plugin/gcp/metrics.go new file mode 100644 index 0000000..cc2c05d --- /dev/null +++ b/plugin/gcp/metrics.go @@ -0,0 +1,26 @@ +package gcp + +import ( + "context" + + monitoring "cloud.google.com/go/monitoring/apiv3/v2" +) + +type CloudMonitoring struct { + client *monitoring.MetricClient + GCP +} + +func (c *CloudMonitoring) InitializeClient(ctx context.Context) error { + + c.GCP.GetCredentials(ctx) + + metricClient, err := monitoring.NewMetricClient(ctx) + if err != nil { + return err + } + + c.client = metricClient + + return nil +} From dc5e6bde49cf9e5274c4d78134ab47b92b474bf9 Mon Sep 17 00:00:00 2001 From: Adnan Gulegulzar Date: Tue, 11 Jun 2024 18:28:30 -0400 Subject: [PATCH 2/4] setup metric client to gather metric datapoints --- plugin/gcp/README.md | 28 +++++++++++++------ plugin/gcp/metrics.go | 57 ++++++++++++++++++++++++++++++++++++++ plugin/gcp/metrics_test.go | 43 ++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 plugin/gcp/metrics_test.go diff --git a/plugin/gcp/README.md b/plugin/gcp/README.md index 18403e9..04161aa 100644 --- a/plugin/gcp/README.md +++ b/plugin/gcp/README.md @@ -15,6 +15,18 @@ - +## Metric Request + +| Component | Description| +|-----------|------------| +| Name | "projects/``" | +|Filter| metric name and resource(instance,device) | +|Interval| start and end time for the metrics | +|Aggregation| reformat for each data point | + + + + TODO: @@ -22,15 +34,15 @@ TODO: - Close metrics client - Metrics to collect: - - "CPUUtilization", + - "CPUUtilization" ~> `compute.googleapis.com/instance/cpu/utilization` - - "NetworkIn", - - "NetworkOut", + - "NetworkIn" ~> `compute.googleapis.com/instance/network/received_bytes_count` + - "NetworkOut" ~> `compute.googleapis.com/instance/network/sent_bytes_count` - - "mem_used_percent", + - "mem_used_percent" ~> `compute.googleapis.com/instance/memory/balloon/ram_size`/`compute.googleapis.com/instance/memory/balloon/ram_used` - - "VolumeReadBytes", - - "VolumeWriteBytes", + - "VolumeReadBytes" ~> `compute.googleapis.com/instance/disk/read_bytes_count` + - "VolumeWriteBytes" ~> `compute.googleapis.com/instance/disk/write_bytes_count` - - "VolumeReadOps", - - "VolumeWriteOps", \ No newline at end of file + - "VolumeReadOps" ~> `compute.googleapis.com/instance/disk/read_ops_count` + - "VolumeWriteOps" ~> `compute.googleapis.com/instance/disk/write_ops_count` \ No newline at end of file diff --git a/plugin/gcp/metrics.go b/plugin/gcp/metrics.go index cc2c05d..f9e78d4 100644 --- a/plugin/gcp/metrics.go +++ b/plugin/gcp/metrics.go @@ -2,8 +2,13 @@ package gcp import ( "context" + "fmt" + "time" monitoring "cloud.google.com/go/monitoring/apiv3/v2" + "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" ) type CloudMonitoring struct { @@ -11,6 +16,12 @@ type CloudMonitoring struct { GCP } +func NewCloudMonitoring(scopes []string) *CloudMonitoring { + return &CloudMonitoring{ + GCP: NewGCP(scopes), + } +} + func (c *CloudMonitoring) InitializeClient(ctx context.Context) error { c.GCP.GetCredentials(ctx) @@ -24,3 +35,49 @@ func (c *CloudMonitoring) InitializeClient(ctx context.Context) error { return nil } + +func (c *CloudMonitoring) CloseClient() error { + err := c.client.Close() + if err != nil { + return err + } + return nil +} + +func (c *CloudMonitoring) NewMetricRequest(metricName string, instanceID string) *monitoringpb.ListTimeSeriesRequest { + + endtime := time.Now() + starttime := endtime.Add(-24 * 1 * time.Hour) // 24 hours before current time + + request := &monitoringpb.ListTimeSeriesRequest{ + Name: fmt.Sprintf("projects/%s", c.ProjectID), + Filter: fmt.Sprintf( + `metric.type="%s" AND resource.labels.instance_id="%s"`, + metricName, + instanceID, + ), + Interval: &monitoringpb.TimeInterval{ + EndTime: timestamppb.New(endtime), + StartTime: timestamppb.New(starttime), + }, + Aggregation: &monitoringpb.Aggregation{ + AlignmentPeriod: durationpb.New(time.Minute), + }, + View: monitoringpb.ListTimeSeriesRequest_FULL, + } + + return request +} + +func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest) *monitoringpb.TimeSeries { + + it := c.client.ListTimeSeries(context.Background(), request) + + resp, err := it.Next() + if err != nil { + panic(err) + } + + return resp + +} diff --git a/plugin/gcp/metrics_test.go b/plugin/gcp/metrics_test.go new file mode 100644 index 0000000..310ae60 --- /dev/null +++ b/plugin/gcp/metrics_test.go @@ -0,0 +1,43 @@ +package gcp + +import ( + "context" + "log" + "os" + "testing" +) + +// run this test as +// +// `TEST_INSTANCE_ID="" make testgcp` +func TestGetMetrics(t *testing.T) { + + id := os.Getenv("TEST_INSTANCE_ID") + + log.Printf("running %s", t.Name()) + metric := NewCloudMonitoring( + []string{ + "https://www.googleapis.com/auth/monitoring.read", + }, + ) + err := metric.InitializeClient(context.Background()) + if err != nil { + t.Errorf("[%s]: %s", t.Name(), err.Error()) + } + + request := metric.NewMetricRequest( + "compute.googleapis.com/instance/cpu/utilization", + id, + ) + + resp := metric.GetMetric(request) + + log.Printf("metrics: %s", resp.GetMetric().String()) + log.Printf("resource: %s", resp.GetResource().String()) + + for _, point := range resp.Points { + log.Printf("Point : %s", point.String()) + } + + metric.CloseClient() +} From 62f5626b3027519ffaf2e4f4263dcd931c967f98 Mon Sep 17 00:00:00 2001 From: Adnan Gulegulzar Date: Tue, 11 Jun 2024 18:29:44 -0400 Subject: [PATCH 3/4] readme update --- plugin/gcp/README.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/plugin/gcp/README.md b/plugin/gcp/README.md index 04161aa..89618d0 100644 --- a/plugin/gcp/README.md +++ b/plugin/gcp/README.md @@ -15,19 +15,6 @@ - -## Metric Request - -| Component | Description| -|-----------|------------| -| Name | "projects/``" | -|Filter| metric name and resource(instance,device) | -|Interval| start and end time for the metrics | -|Aggregation| reformat for each data point | - - - - - TODO: - Metrics Client From 852ab728be5456524180677c7266bd91088ddebf Mon Sep 17 00:00:00 2001 From: Adnan Gulegulzar Date: Thu, 13 Jun 2024 23:58:17 -0400 Subject: [PATCH 4/4] finalized metrics for PR --- Makefile | 2 +- plugin/gcp/metrics.go | 20 +++++++++++++------- plugin/gcp/metrics_test.go | 20 ++++++++++++++++---- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 0817c00..e613252 100644 --- a/Makefile +++ b/Makefile @@ -13,4 +13,4 @@ goreleaser: clean # test components of plugin/gcp testgcp: - go test -v ./plugin/gcp \ No newline at end of file + go test -v ./plugin/gcp -count=1 \ No newline at end of file diff --git a/plugin/gcp/metrics.go b/plugin/gcp/metrics.go index f9e78d4..9790a15 100644 --- a/plugin/gcp/metrics.go +++ b/plugin/gcp/metrics.go @@ -44,10 +44,13 @@ func (c *CloudMonitoring) CloseClient() error { return nil } -func (c *CloudMonitoring) NewMetricRequest(metricName string, instanceID string) *monitoringpb.ListTimeSeriesRequest { - - endtime := time.Now() - starttime := endtime.Add(-24 * 1 * time.Hour) // 24 hours before current time +func (c *CloudMonitoring) NewInstanceMetricRequest( + metricName string, // fully qualified name of the metric + instanceID string, // compute instance ID + startTime time.Time, // start time of requested time series + endTime time.Time, // end time of requested time series + periodInSeconds int64, // period, for which the datapoints will be aggregated into one, in seconds +) *monitoringpb.ListTimeSeriesRequest { request := &monitoringpb.ListTimeSeriesRequest{ Name: fmt.Sprintf("projects/%s", c.ProjectID), @@ -57,11 +60,14 @@ func (c *CloudMonitoring) NewMetricRequest(metricName string, instanceID string) instanceID, ), Interval: &monitoringpb.TimeInterval{ - EndTime: timestamppb.New(endtime), - StartTime: timestamppb.New(starttime), + EndTime: timestamppb.New(endTime), + StartTime: timestamppb.New(startTime), }, Aggregation: &monitoringpb.Aggregation{ - AlignmentPeriod: durationpb.New(time.Minute), + AlignmentPeriod: &durationpb.Duration{ + Seconds: periodInSeconds, + }, + PerSeriesAligner: monitoringpb.Aggregation_ALIGN_MEAN, // will represent all the datapoints in the above period, with a mean }, View: monitoringpb.ListTimeSeriesRequest_FULL, } diff --git a/plugin/gcp/metrics_test.go b/plugin/gcp/metrics_test.go index 310ae60..37a6244 100644 --- a/plugin/gcp/metrics_test.go +++ b/plugin/gcp/metrics_test.go @@ -5,6 +5,7 @@ import ( "log" "os" "testing" + "time" ) // run this test as @@ -12,9 +13,14 @@ import ( // `TEST_INSTANCE_ID="" make testgcp` func TestGetMetrics(t *testing.T) { + //test variables id := os.Getenv("TEST_INSTANCE_ID") + endtime := time.Now() + starttime := endtime.Add(-24 * 1 * time.Hour) // 24 hours before current time log.Printf("running %s", t.Name()) + + // creating and initializing client metric := NewCloudMonitoring( []string{ "https://www.googleapis.com/auth/monitoring.read", @@ -25,19 +31,25 @@ func TestGetMetrics(t *testing.T) { t.Errorf("[%s]: %s", t.Name(), err.Error()) } - request := metric.NewMetricRequest( + // creating the metric request for the instance + request := metric.NewInstanceMetricRequest( "compute.googleapis.com/instance/cpu/utilization", id, + starttime, + endtime, + 60, ) + // execute the request resp := metric.GetMetric(request) log.Printf("metrics: %s", resp.GetMetric().String()) log.Printf("resource: %s", resp.GetResource().String()) + log.Printf("# of points: %d", len(resp.Points)) - for _, point := range resp.Points { - log.Printf("Point : %s", point.String()) - } + // for _, point := range resp.Points { + // log.Printf("Point : %.10f", point.GetValue().GetDoubleValue()) + // } metric.CloseClient() }