diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8654c83..91b07b4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -46,26 +46,26 @@ jobs: with: name: new_tag path: new_tag.txt - release: - runs-on: ubuntu-latest - needs: - - tag - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GH_TOKEN }} - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: stable - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v5 - with: - distribution: goreleaser - version: latest - args: release --clean - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# release: +# runs-on: ubuntu-latest +# needs: +# - tag +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# token: ${{ secrets.GH_TOKEN }} +# - name: Set up Go +# uses: actions/setup-go@v4 +# with: +# go-version: stable +# - name: Run GoReleaser +# uses: goreleaser/goreleaser-action@v5 +# with: +# distribution: goreleaser +# version: latest +# args: release --clean +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/go.mod b/go.mod index f260518..1f419ec 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,8 @@ go 1.21.5 require ( cloud.google.com/go/compute v1.25.1 cloud.google.com/go/monitoring v1.18.0 - github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.6 github.com/google/uuid v1.6.0 - github.com/kaytu-io/kaytu v0.10.5 + github.com/kaytu-io/kaytu v0.14.0-rc.0.0.20240619114201-51f5733d10b6 golang.org/x/oauth2 v0.21.0 google.golang.org/api v0.169.0 google.golang.org/protobuf v1.34.2 @@ -15,7 +14,6 @@ require ( require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect - github.com/aws/smithy-go v1.20.2 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 66df2df..ea7dd46 100644 --- a/go.sum +++ b/go.sum @@ -8,10 +8,6 @@ cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1h cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= cloud.google.com/go/monitoring v1.18.0/go.mod h1:c92vVBCeq/OB4Ioyo+NbN2U7tlg5ZH41PZcdvfc+Lcg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.6 h1:UVjxYe8VGpwXYcmBcciBHlQrNssdEvntXCPWmnRR15U= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.6/go.mod h1:4V6VDA0kZavRn71+sLpVna75oobnlG+gwtnNcBwZhu4= -github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= -github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -65,8 +61,8 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/kaytu-io/kaytu v0.10.5 h1:EGrvEQTjnIC06ix2rUt0RBlxHjS50pHKifSVQJOH1Ho= -github.com/kaytu-io/kaytu v0.10.5/go.mod h1:BhFDEm+6ckgusHsrDCdgSH+b/wMhuRsq29L8/PJe2Sk= +github.com/kaytu-io/kaytu v0.14.0-rc.0.0.20240619114201-51f5733d10b6 h1:OyrvbIttya/35K+6N1tyI953BlKclIkHGWIZ/IPYheg= +github.com/kaytu-io/kaytu v0.14.0-rc.0.0.20240619114201-51f5733d10b6/go.mod h1:csDLaCgTnp/D9SZuo55KjOzp/BYfkhkeSkGuEiCljtc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/plugin/gcp/compute.go b/plugin/gcp/compute.go index 9f9e9ba..848e8d8 100644 --- a/plugin/gcp/compute.go +++ b/plugin/gcp/compute.go @@ -13,7 +13,8 @@ import ( ) type Compute struct { - client *compute.InstancesClient + instancesClient *compute.InstancesClient + machineTypeClient *compute.MachineTypesClient GCP } @@ -38,15 +39,28 @@ func (c *Compute) InitializeClient(ctx context.Context) error { return err } + machineTypeClient, err := compute.NewMachineTypesRESTClient( + ctx, + option.WithCredentials(c.GCP.credentials), + ) + if err != nil { + return err + } + // log.Println(instancesClient) - c.client = instancesClient + c.instancesClient = instancesClient + c.machineTypeClient = machineTypeClient return nil } func (c *Compute) CloseClient() error { - err := c.client.Close() + err := c.instancesClient.Close() + if err != nil { + return err + } + err = c.machineTypeClient.Close() if err != nil { return err } @@ -59,7 +73,7 @@ func (c *Compute) ListAllInstances() error { Project: c.ProjectID, } - it := c.client.AggregatedList(context.Background(), req) + it := c.instancesClient.AggregatedList(context.Background(), req) log.Println("instances found: ") @@ -92,7 +106,7 @@ func (c *Compute) GetAllInstances() ([]*computepb.Instance, error) { Project: c.ProjectID, } - it := c.client.AggregatedList(context.Background(), req) + it := c.instancesClient.AggregatedList(context.Background(), req) log.Println("instances found: ") @@ -121,3 +135,22 @@ func (c *Compute) GetAllInstances() ([]*computepb.Instance, error) { } return allInstances, nil } + +func (c *Compute) GetMemory(InstanceMachineType string, zone string) (*int32, error) { + + request := &computepb.GetMachineTypeRequest{ + Project: c.ProjectID, + MachineType: InstanceMachineType, + Zone: zone, + } + + machineType, err := c.machineTypeClient.Get(context.Background(), request) + if err != nil { + return nil, err + } + + memory := machineType.GetMemoryMb() + + return &memory, nil + +} diff --git a/plugin/gcp/compute_test.go b/plugin/gcp/compute_test.go index 36f5fc8..dd7a8cc 100644 --- a/plugin/gcp/compute_test.go +++ b/plugin/gcp/compute_test.go @@ -3,6 +3,7 @@ package gcp import ( "context" "log" + "os" "testing" ) @@ -59,3 +60,29 @@ func TestGetAllInstances(t *testing.T) { compute.CloseClient() } + +// TEST_INSTANCE_ZONE="us-east1-b" TEST_INSTANCE_MACHINE_TYPE="e2-micro" TEST_INSTANCE_ID="7828543314219019363" make testgcp +func TestGetMemory(t *testing.T) { + + machineType := os.Getenv("TEST_INSTANCE_MACHINE_TYPE") + zone := os.Getenv("TEST_INSTANCE_ZONE") + + log.Printf("running %s", t.Name()) + compute := NewCompute( + []string{ + "https://www.googleapis.com/auth/compute.readonly", + }, + ) + err := compute.InitializeClient(context.Background()) + if err != nil { + t.Errorf("[%s]: %s", t.Name(), err.Error()) + } + + memory, err := compute.GetMemory(machineType, zone) + if err != nil { + t.Errorf("[%s]: %s", t.Name(), err.Error()) + } + + log.Printf("Memory : %d", &memory) + +} diff --git a/plugin/gcp/config.go b/plugin/gcp/config.go index 144723f..694b968 100644 --- a/plugin/gcp/config.go +++ b/plugin/gcp/config.go @@ -21,7 +21,9 @@ func NewGCP(scopes []string) GCP { } func (g *GCP) GetCredentials(ctx context.Context) error { - credentials, err := google.FindDefaultCredentials( + var err error + + g.credentials, err = google.FindDefaultCredentials( ctx, g.Scopes..., ) @@ -29,9 +31,18 @@ func (g *GCP) GetCredentials(ctx context.Context) error { return err } - json.Unmarshal(credentials.JSON, g) //this will store project id from credentials + g.ProjectID = g.credentials.ProjectID + + json.Unmarshal(g.credentials.JSON, g) //this will store project id from credentials - g.credentials = credentials - // log.Println(g.ProjectID) return nil } + +func (g *GCP) Identify() map[string]string { + + identification := map[string]string{ + "project_id": g.ProjectID, + } + + return identification +} diff --git a/plugin/gcp/config_test.go b/plugin/gcp/config_test.go new file mode 100644 index 0000000..a9904d7 --- /dev/null +++ b/plugin/gcp/config_test.go @@ -0,0 +1,22 @@ +package gcp + +import ( + "log" + "testing" +) + +func TestIdentify(t *testing.T) { + + test_project_id := "test-project-id" + gcp := GCP{ + ProjectID: test_project_id, + } + + identification := gcp.Identify() + log.Println(identification) + + if identification["project_id"] != test_project_id { + t.Error("TestIdentify failed") + } + +} diff --git a/plugin/gcp/metrics.go b/plugin/gcp/metrics.go index 9790a15..911c203 100644 --- a/plugin/gcp/metrics.go +++ b/plugin/gcp/metrics.go @@ -3,12 +3,11 @@ package gcp import ( "context" "fmt" - "time" + "github.com/kaytu-io/plugin-gcp/plugin/kaytu" + "google.golang.org/api/iterator" monitoring "cloud.google.com/go/monitoring/apiv3/v2" "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/timestamppb" ) type CloudMonitoring struct { @@ -44,46 +43,51 @@ func (c *CloudMonitoring) CloseClient() error { return nil } -func (c *CloudMonitoring) NewInstanceMetricRequest( - metricName string, // fully qualified name of the metric - instanceID string, // compute instance ID - startTime time.Time, // start time of requested time series - endTime time.Time, // end time of requested time series - periodInSeconds int64, // period, for which the datapoints will be aggregated into one, in seconds +func (c *CloudMonitoring) NewTimeSeriesRequest( + filter string, // filter for time series metric, containing metric name and resource label + interval *monitoringpb.TimeInterval, // interval containing start and end time of the requested time series + aggregation *monitoringpb.Aggregation, // operations to perform on time series data before returning ) *monitoringpb.ListTimeSeriesRequest { - request := &monitoringpb.ListTimeSeriesRequest{ - Name: fmt.Sprintf("projects/%s", c.ProjectID), - Filter: fmt.Sprintf( - `metric.type="%s" AND resource.labels.instance_id="%s"`, - metricName, - instanceID, - ), - Interval: &monitoringpb.TimeInterval{ - EndTime: timestamppb.New(endTime), - StartTime: timestamppb.New(startTime), - }, - Aggregation: &monitoringpb.Aggregation{ - AlignmentPeriod: &durationpb.Duration{ - Seconds: periodInSeconds, - }, - PerSeriesAligner: monitoringpb.Aggregation_ALIGN_MEAN, // will represent all the datapoints in the above period, with a mean - }, - View: monitoringpb.ListTimeSeriesRequest_FULL, + return &monitoringpb.ListTimeSeriesRequest{ + Name: fmt.Sprintf("projects/%s", c.ProjectID), + Filter: filter, + Interval: interval, + Aggregation: aggregation, + View: monitoringpb.ListTimeSeriesRequest_FULL, } - return request } -func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest) *monitoringpb.TimeSeries { +func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest) ([]kaytu.Datapoint, error) { + var dps []kaytu.Datapoint it := c.client.ListTimeSeries(context.Background(), request) - - resp, err := it.Next() - if err != nil { - panic(err) + for { + resp, err := it.Next() + + if err != nil { + if err == iterator.Done { + break + } else { + return nil, err + } + } + dps = append(dps, convertDatapoints(resp)...) } - return resp + return dps, nil } + +func convertDatapoints(resp *monitoringpb.TimeSeries) []kaytu.Datapoint { + var dps []kaytu.Datapoint + for _, dp := range resp.GetPoints() { + dps = append(dps, kaytu.Datapoint{ + Value: dp.GetValue().GetDoubleValue(), + StartTime: dp.GetInterval().GetStartTime().AsTime(), + EndTime: dp.GetInterval().GetEndTime().AsTime(), + }) + } + return dps +} diff --git a/plugin/gcp/metrics_test.go b/plugin/gcp/metrics_test.go index 37a6244..8f2a388 100644 --- a/plugin/gcp/metrics_test.go +++ b/plugin/gcp/metrics_test.go @@ -2,10 +2,14 @@ package gcp import ( "context" - "log" + "fmt" "os" "testing" "time" + + "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" ) // run this test as @@ -15,10 +19,10 @@ func TestGetMetrics(t *testing.T) { //test variables id := os.Getenv("TEST_INSTANCE_ID") - endtime := time.Now() - starttime := endtime.Add(-24 * 1 * time.Hour) // 24 hours before current time + endTime := time.Now() + startTime := endTime.Add(-24 * 1 * time.Hour) // 24 hours before current time - log.Printf("running %s", t.Name()) + t.Logf("running %s", t.Name()) // creating and initializing client metric := NewCloudMonitoring( @@ -32,23 +36,38 @@ func TestGetMetrics(t *testing.T) { } // creating the metric request for the instance - request := metric.NewInstanceMetricRequest( - "compute.googleapis.com/instance/cpu/utilization", - id, - starttime, - endtime, - 60, + memoryRequest := metric.NewTimeSeriesRequest( + fmt.Sprintf( + `metric.type="%s" AND resource.labels.instance_id="%s"`, + "compute.googleapis.com/instance/memory/balloon/ram_used", + id, + ), + &monitoringpb.TimeInterval{ + EndTime: timestamppb.New(endTime), + StartTime: timestamppb.New(startTime), + }, + &monitoringpb.Aggregation{ + AlignmentPeriod: &durationpb.Duration{ + Seconds: 60, + }, + PerSeriesAligner: monitoringpb.Aggregation_ALIGN_NONE, // will represent all the datapoints in the above period, with a mean + }, ) // execute the request - resp := metric.GetMetric(request) + resp, err := metric.GetMetric(memoryRequest) + if err != nil { + t.Error(err) + } - log.Printf("metrics: %s", resp.GetMetric().String()) - log.Printf("resource: %s", resp.GetResource().String()) - log.Printf("# of points: %d", len(resp.Points)) + // log.Printf("metrics: %s", resp.GetMetric().String()) + // log.Printf("resource: %s", resp.GetResource().String()) + t.Logf("datapoints: %d", len(resp)) + // log.Println(resp.Unit) + // log.Printf("Point 1 : %.0f %s", resp.GetPoints()[0].GetValue().GetDoubleValue(), resp.GetUnit()) // for _, point := range resp.Points { - // log.Printf("Point : %.10f", point.GetValue().GetDoubleValue()) + // log.Printf("Point : %.0f", point.GetValue().GetDoubleValue()) // } metric.CloseClient() diff --git a/plugin/kaytu/compute_instance.go b/plugin/kaytu/compute_instance.go index b71a2b3..2c11927 100644 --- a/plugin/kaytu/compute_instance.go +++ b/plugin/kaytu/compute_instance.go @@ -1,8 +1,6 @@ package kaytu -import ( - "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" -) +import "time" type GcpComputeInstance struct { HashedInstanceId string `json:"hashedInstanceId"` @@ -31,20 +29,26 @@ type GcpComputeInstanceRightsizingRecommendation struct { } type GcpComputeInstanceWastageRequest struct { - RequestId *string `json:"requestId"` - CliVersion *string `json:"cliVersion"` - Identification map[string]string `json:"identification"` - Instance GcpComputeInstance `json:"instance"` - Metrics map[string][]*monitoringpb.Point `json:"metrics"` - Region string `json:"region"` - Preferences map[string]*string `json:"preferences"` - Loading bool `json:"loading"` + RequestId *string `json:"requestId"` + CliVersion *string `json:"cliVersion"` + Identification map[string]string `json:"identification"` + Instance GcpComputeInstance `json:"instance"` + Metrics map[string][]Datapoint `json:"metrics"` + Region string `json:"region"` + Preferences map[string]*string `json:"preferences"` + Loading bool `json:"loading"` } type GcpComputeInstanceWastageResponse struct { RightSizing GcpComputeInstanceRightsizingRecommendation `json:"rightSizing"` } +type Datapoint struct { + StartTime time.Time + EndTime time.Time + Value float64 +} + type Usage struct { Avg *float64 Min *float64 diff --git a/plugin/kaytu/request.go b/plugin/kaytu/request.go index c117f22..c2a3ea2 100644 --- a/plugin/kaytu/request.go +++ b/plugin/kaytu/request.go @@ -17,7 +17,6 @@ func Ec2InstanceWastageRequest(reqBody GcpComputeInstanceWastageRequest, token s return nil, err } req, err := http.NewRequest("POST", "https://api.kaytu.io/kaytu/wastage/api/v1/wastage/gcp-compute", bytes.NewBuffer(payloadEncoded)) - //req, err := http.NewRequest("POST", "http://localhost:8000/api/v1/wastage/ec2-instance", bytes.NewBuffer(payloadEncoded)) if err != nil { return nil, fmt.Errorf("[ec2-instance]: %v", err) } diff --git a/plugin/preferences/default.go b/plugin/preferences/default.go index 04568bb..15de420 100644 --- a/plugin/preferences/default.go +++ b/plugin/preferences/default.go @@ -1,5 +1,15 @@ package preferences -import "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" +import ( + "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" + "google.golang.org/protobuf/types/known/wrapperspb" +) -var DefaultComputeEnginePreferences = []*golang.PreferenceItem{} +var DefaultComputeEnginePreferences = []*golang.PreferenceItem{ + {Service: "ComputeInstance", Key: "vCPU", IsNumber: true}, + //{Service: "ComputeInstance", Key: "Region", Pinned: true}, + {Service: "ComputeInstance", Key: "MachineFamily", Pinned: true}, + {Service: "ComputeInstance", Key: "MemoryGB", Alias: "Memory", IsNumber: true, Unit: "GiB"}, + {Service: "ComputeInstance", Key: "CPUBreathingRoom", IsNumber: true, Value: wrapperspb.String("10"), PreventPinning: true, Unit: "%"}, + {Service: "ComputeInstance", Key: "MemoryBreathingRoom", IsNumber: true, Value: wrapperspb.String("10"), PreventPinning: true, Unit: "%"}, +} diff --git a/plugin/processor/compute_instance/compute_instance.go b/plugin/processor/compute_instance/compute_instance.go index e095333..8449ab3 100644 --- a/plugin/processor/compute_instance/compute_instance.go +++ b/plugin/processor/compute_instance/compute_instance.go @@ -10,8 +10,8 @@ import ( ) type ComputeInstanceProcessor struct { - provider *gcp.Compute - // identification map[string]string + provider *gcp.Compute + metricProvider *gcp.CloudMonitoring items util.ConcurrentMap[string, ComputeInstanceItem] publishOptimizationItem func(item *golang.ChartOptimizationItem) publishResultSummary func(summary *golang.ResultSummary) @@ -21,14 +21,16 @@ type ComputeInstanceProcessor struct { func NewComputeInstanceProcessor( prv *gcp.Compute, + metricPrv *gcp.CloudMonitoring, publishOptimizationItem func(item *golang.ChartOptimizationItem), publishResultSummary func(summary *golang.ResultSummary), kaytuAcccessToken string, jobQueue *sdk.JobQueue, ) *ComputeInstanceProcessor { + log.Println("creating processor") r := &ComputeInstanceProcessor{ - provider: prv, - // identification: identification, + provider: prv, + metricProvider: metricPrv, items: util.NewMap[string, ComputeInstanceItem](), publishOptimizationItem: publishOptimizationItem, publishResultSummary: publishResultSummary, @@ -37,7 +39,6 @@ func NewComputeInstanceProcessor( // configuration: configurations, // lazyloadCounter: lazyloadCounter, } - log.Println("creating processor") jobQueue.Push(NewListComputeInstancesJob(r)) return r @@ -50,3 +51,7 @@ func (m *ComputeInstanceProcessor) ReEvaluate(id string, items []*golang.Prefere // m.items.Set(id, v) // m.jobQueue.Push(NewOptimizeEC2InstanceJob(m, v)) } + +func (p *ComputeInstanceProcessor) ExportNonInteractive() *golang.NonInteractiveExport { + return nil +} diff --git a/plugin/processor/compute_instance/compute_instance_item.go b/plugin/processor/compute_instance/compute_instance_item.go index 3883ab6..5d65319 100644 --- a/plugin/processor/compute_instance/compute_instance_item.go +++ b/plugin/processor/compute_instance/compute_instance_item.go @@ -1,67 +1,172 @@ package compute_instance import ( - "github.com/google/uuid" + "fmt" + "maps" + "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" + "github.com/kaytu-io/kaytu/pkg/utils" + "github.com/kaytu-io/plugin-gcp/plugin/kaytu" "google.golang.org/protobuf/types/known/wrapperspb" ) type ComputeInstanceItem struct { + ProjectId string Name string Id string MachineType string Region string + Platform string OptimizationLoading bool Preferences []*golang.PreferenceItem Skipped bool LazyLoadingEnabled bool SkipReason string - // Wastage kaytu.EC2InstanceWastageResponse + Metrics map[string][]kaytu.Datapoint + Wastage kaytu.GcpComputeInstanceWastageResponse } -// func (i ComputeInstanceItem) ToOptimizationItem() *golang.OptimizationItem { -// oi := &golang.OptimizationItem{ -// Id: i.Id, -// Name: i.Name, -// ResourceType: i.MachineType, -// Region: i.Region, -// Devices: nil, -// Preferences: i.Preferences, -// Description: "description placeholder", -// Loading: i.OptimizationLoading, -// Skipped: i.Skipped, -// SkipReason: i.SkipReason, -// LazyLoadingEnabled: i.LazyLoadingEnabled, -// } - -// // if i.Instance.PlatformDetails != nil { -// // oi.Platform = *i.Instance.PlatformDetails -// // } -// if oi.Name == "" { -// oi.Name = string(i.Name) -// } - -// return oi -// } +func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[string]*golang.Properties) { + row := golang.ChartRow{ + RowId: i.Id, + Values: make(map[string]*golang.ChartRowItem), + } + row.RowId = i.Id + + row.Values["resource_id"] = &golang.ChartRowItem{ + Value: i.Id, + } + row.Values["resource_name"] = &golang.ChartRowItem{ + Value: i.Name, + } + row.Values["resource_type"] = &golang.ChartRowItem{ + Value: "Compute Instance", + } + + row.Values["current_cost"] = &golang.ChartRowItem{ + Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost), + } + + ZoneProperty := &golang.Property{ + Key: "Zone", + Current: i.Wastage.RightSizing.Current.Zone, + } + + MachineTypeProperty := &golang.Property{ + Key: "Machine Type", + Current: i.Wastage.RightSizing.Current.MachineType, + } + MachineFamilyProperty := &golang.Property{ + Key: "Machine Family", + Current: i.Wastage.RightSizing.Current.MachineFamily, + } + CPUProperty := &golang.Property{ + Key: " CPU", + Current: fmt.Sprintf("%d", i.Wastage.RightSizing.Current.CPU), + } + + memoryProperty := &golang.Property{ + Key: " MemoryMB", + Current: fmt.Sprintf("%d MB", i.Wastage.RightSizing.Current.MemoryMb), + Average: utils.Percentage(i.Wastage.RightSizing.Memory.Avg), + Max: utils.Percentage(i.Wastage.RightSizing.Memory.Max), + } + + row.Values["project_id"] = &golang.ChartRowItem{ + Value: i.ProjectId, + } + + row.Values["current_cost"] = &golang.ChartRowItem{ + Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost), + } + + if i.Wastage.RightSizing.Recommended != nil { + row.Values["right_sized_cost"] = &golang.ChartRowItem{ + Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Recommended.Cost), + } + row.Values["savings"] = &golang.ChartRowItem{ + Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost - i.Wastage.RightSizing.Recommended.Cost), + } + ZoneProperty.Recommended = i.Wastage.RightSizing.Recommended.Zone + MachineTypeProperty.Recommended = i.Wastage.RightSizing.Recommended.MachineType + CPUProperty.Recommended = fmt.Sprintf("%d", i.Wastage.RightSizing.Recommended.CPU) + memoryProperty.Recommended = fmt.Sprintf("%d MB", i.Wastage.RightSizing.Recommended.MemoryMb) + } + + props := make(map[string]*golang.Properties) + properties := &golang.Properties{} + + properties.Properties = append(properties.Properties, ZoneProperty) + properties.Properties = append(properties.Properties, MachineTypeProperty) + properties.Properties = append(properties.Properties, MachineFamilyProperty) + properties.Properties = append(properties.Properties, &golang.Property{ + Key: "Compute", + }) + properties.Properties = append(properties.Properties, CPUProperty) + properties.Properties = append(properties.Properties, memoryProperty) + + props[i.Id] = properties + + return &row, props +} + +func (i ComputeInstanceItem) Devices() ([]*golang.ChartRow, map[string]*golang.Properties) { + + var deviceRows []*golang.ChartRow + deviceProps := make(map[string]*golang.Properties) + + instanceRows, instanceProps := i.ComputeInstanceDevice() + + deviceRows = append(deviceRows, instanceRows) + maps.Copy(deviceProps, instanceProps) + + return deviceRows, deviceProps +} func (i ComputeInstanceItem) ToOptimizationItem() *golang.ChartOptimizationItem { + deviceRows, deviceProps := i.Devices() + + status := "" + if i.Wastage.RightSizing.Recommended != nil { + totalSaving := 0.0 + totalCurrentCost := 0.0 + totalSaving += i.Wastage.RightSizing.Current.Cost - i.Wastage.RightSizing.Recommended.Cost + totalCurrentCost += i.Wastage.RightSizing.Current.Cost + status = fmt.Sprintf("%s (%.2f%%)", utils.FormatPriceFloat(totalSaving), (totalSaving/totalCurrentCost)*100) + } + chartrow := &golang.ChartRow{ - RowId: uuid.New().String(), + RowId: i.Id, Values: map[string]*golang.ChartRowItem{ - "instance_name": { + "x_kaytu_right_arrow": { + Value: "→", + }, + "resource_id": { + Value: i.Id, + }, + "resource_name": { Value: i.Name, }, + "resource_type": { + Value: i.MachineType, + }, + "region": { + Value: i.Region, + }, + "platform": { + Value: i.Platform, + }, + "total_saving": { + Value: status, + }, }, } - oi := &golang.ChartOptimizationItem{ - OverviewChartRow: chartrow, - // Id: i.Id, - // Name: i.Name, - // ResourceType: i.MachineType, - // Region: i.Region, - // Devices: nil, + coi := &golang.ChartOptimizationItem{ + OverviewChartRow: chartrow, + DevicesChartRows: deviceRows, + DevicesProperties: deviceProps, Preferences: i.Preferences, Description: "description placeholder", Loading: i.OptimizationLoading, @@ -70,12 +175,5 @@ func (i ComputeInstanceItem) ToOptimizationItem() *golang.ChartOptimizationItem LazyLoadingEnabled: i.LazyLoadingEnabled, } - // if i.Instance.PlatformDetails != nil { - // oi.Platform = *i.Instance.PlatformDetails - // } - // if oi.Name == "" { - // oi.Name = string(i.Name) - // } - - return oi + return coi } diff --git a/plugin/processor/compute_instance/job_compute_instance_list.go b/plugin/processor/compute_instance/job_compute_instance_list.go index aa406ce..7fd26a7 100644 --- a/plugin/processor/compute_instance/job_compute_instance_list.go +++ b/plugin/processor/compute_instance/job_compute_instance_list.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/kaytu-io/plugin-gcp/plugin/preferences" + util "github.com/kaytu-io/plugin-gcp/utils" ) type ListComputeInstancesJob struct { @@ -27,15 +28,10 @@ func (job *ListComputeInstancesJob) Description() string { } -func (job *ListComputeInstancesJob) Run() error { +func (job *ListComputeInstancesJob) Run(ctx context.Context) error { log.Println("Running list compute instance job") - err := job.processor.provider.InitializeClient(context.Background()) - if err != nil { - return err - } - instances, err := job.processor.provider.GetAllInstances() if err != nil { return err @@ -45,15 +41,18 @@ func (job *ListComputeInstancesJob) Run() error { for _, instance := range instances { oi := ComputeInstanceItem{ + ProjectId: job.processor.provider.ProjectID, Name: *instance.Name, Id: strconv.FormatUint(instance.GetId(), 10), - MachineType: instance.GetMachineType(), - Region: instance.GetZone(), + MachineType: util.TrimmedString(*instance.MachineType, "/"), + Region: util.TrimmedString(*instance.Zone, "/"), + Platform: instance.GetCpuPlatform(), OptimizationLoading: false, Preferences: preferences.DefaultComputeEnginePreferences, Skipped: false, LazyLoadingEnabled: false, SkipReason: "NA", + Metrics: nil, } log.Printf("OI instance: %s", oi.Name) @@ -62,8 +61,9 @@ func (job *ListComputeInstancesJob) Run() error { job.processor.publishOptimizationItem(oi.ToOptimizationItem()) } - if err = job.processor.provider.CloseClient(); err != nil { - return err + for _, instance := range instances { + + job.processor.jobQueue.Push(NewGetComputeInstanceMetricsJob(job.processor, instance)) } return nil diff --git a/plugin/processor/compute_instance/job_get_compute_instance_metrics.go b/plugin/processor/compute_instance/job_get_compute_instance_metrics.go new file mode 100644 index 0000000..7890ab7 --- /dev/null +++ b/plugin/processor/compute_instance/job_get_compute_instance_metrics.go @@ -0,0 +1,128 @@ +package compute_instance + +import ( + "context" + "fmt" + "log" + "strconv" + "time" + + "github.com/kaytu-io/plugin-gcp/plugin/kaytu" + + "cloud.google.com/go/compute/apiv1/computepb" + "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" + "github.com/kaytu-io/plugin-gcp/plugin/preferences" + util "github.com/kaytu-io/plugin-gcp/utils" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type GetComputeInstanceMetricsJob struct { + processor *ComputeInstanceProcessor + instance *computepb.Instance +} + +func NewGetComputeInstanceMetricsJob(processor *ComputeInstanceProcessor, instance *computepb.Instance) *GetComputeInstanceMetricsJob { + return &GetComputeInstanceMetricsJob{ + processor: processor, + instance: instance, + } +} + +func (job *GetComputeInstanceMetricsJob) Id() string { + return fmt.Sprintf("get_compute_instance_metrics_%d", job.instance.GetId()) +} + +func (job *GetComputeInstanceMetricsJob) Description() string { + return fmt.Sprintf("Get metrics for compute instance: %d", job.instance.GetId()) + +} + +func (job *GetComputeInstanceMetricsJob) Run(ctx context.Context) error { + + endTime := time.Now() // end time of requested time series + startTime := endTime.Add(-24 * 1 * time.Hour) // start time of requested time series + + // metricName string, + // instanceID string, + // startTime time.Time, + // endTime time.Time, + // periodInSeconds int64, + cpuRequest := job.processor.metricProvider.NewTimeSeriesRequest( + fmt.Sprintf( + `metric.type="%s" AND resource.labels.instance_id="%s"`, + "compute.googleapis.com/instance/cpu/utilization", // fully qualified name of the metric + fmt.Sprint(job.instance.GetId()), // compute instance ID + ), + &monitoringpb.TimeInterval{ + EndTime: timestamppb.New(endTime), + StartTime: timestamppb.New(startTime), + }, + &monitoringpb.Aggregation{ + AlignmentPeriod: &durationpb.Duration{ // period, for which the datapoints will be aggregated into one, in seconds + Seconds: 60, + }, + PerSeriesAligner: monitoringpb.Aggregation_ALIGN_MEAN, // will represent all the datapoints in the above period, with a mean + }, + ) + + cpumetric, err := job.processor.metricProvider.GetMetric(cpuRequest) + if err != nil { + return err + } + + memoryRequest := job.processor.metricProvider.NewTimeSeriesRequest( + fmt.Sprintf( + `metric.type="%s" AND resource.labels.instance_id="%s"`, + "compute.googleapis.com/instance/memory/balloon/ram_used", + fmt.Sprint(job.instance.GetId()), + ), + &monitoringpb.TimeInterval{ + EndTime: timestamppb.New(endTime), + StartTime: timestamppb.New(startTime), + }, + &monitoringpb.Aggregation{ + AlignmentPeriod: &durationpb.Duration{ + Seconds: 60, + }, + PerSeriesAligner: monitoringpb.Aggregation_ALIGN_MEAN, // will represent all the datapoints in the above period, with a mean + }, + ) + + memoryMetric, err := job.processor.metricProvider.GetMetric(memoryRequest) + if err != nil { + return err + } + + instanceMetrics := make(map[string][]kaytu.Datapoint) + + instanceMetrics["cpuUtilization"] = cpumetric + instanceMetrics["memoryUtilization"] = memoryMetric + + oi := ComputeInstanceItem{ + ProjectId: job.processor.provider.ProjectID, + Name: *job.instance.Name, + Id: strconv.FormatUint(job.instance.GetId(), 10), + MachineType: util.TrimmedString(*job.instance.MachineType, "/"), + Region: util.TrimmedString(*job.instance.Zone, "/"), + Platform: job.instance.GetCpuPlatform(), + OptimizationLoading: false, + Preferences: preferences.DefaultComputeEnginePreferences, + Skipped: false, + LazyLoadingEnabled: false, + SkipReason: "NA", + Metrics: instanceMetrics, + } + + for k, v := range oi.Metrics { + log.Printf("%s : %d", k, len(v)) + } + + job.processor.items.Set(oi.Id, oi) + job.processor.publishOptimizationItem(oi.ToOptimizationItem()) + + job.processor.jobQueue.Push(NewOptimizeComputeInstancesJob(job.processor, oi)) + + return nil + +} diff --git a/plugin/processor/compute_instance/job_optimize_compute_instance.go b/plugin/processor/compute_instance/job_optimize_compute_instance.go new file mode 100644 index 0000000..9bdb4a9 --- /dev/null +++ b/plugin/processor/compute_instance/job_optimize_compute_instance.go @@ -0,0 +1,79 @@ +package compute_instance + +import ( + "context" + "fmt" + + "github.com/google/uuid" + "github.com/kaytu-io/kaytu/pkg/utils" + "github.com/kaytu-io/kaytu/preferences" + "github.com/kaytu-io/plugin-gcp/plugin/kaytu" + "github.com/kaytu-io/plugin-gcp/plugin/version" +) + +type OptimizeComputeInstancesJob struct { + processor *ComputeInstanceProcessor + item ComputeInstanceItem +} + +func NewOptimizeComputeInstancesJob(processor *ComputeInstanceProcessor, item ComputeInstanceItem) *OptimizeComputeInstancesJob { + return &OptimizeComputeInstancesJob{ + processor: processor, + item: item, + } +} + +func (job *OptimizeComputeInstancesJob) Id() string { + return fmt.Sprintf("optimize_compute_isntance_%s", job.item.Id) +} + +func (job *OptimizeComputeInstancesJob) Description() string { + return fmt.Sprintf("Optimizing %s", job.item.Id) + +} + +func (job *OptimizeComputeInstancesJob) Run(ctx context.Context) error { + + requestId := uuid.NewString() + + request := kaytu.GcpComputeInstanceWastageRequest{ + RequestId: &requestId, + CliVersion: &version.VERSION, + Identification: job.processor.provider.Identify(), + Instance: kaytu.GcpComputeInstance{ + HashedInstanceId: utils.HashString(job.item.Id), + Zone: job.item.Region, + MachineType: job.item.MachineType, + }, + Metrics: job.item.Metrics, + Region: job.item.Region, + Preferences: preferences.Export(job.item.Preferences), + Loading: false, + } + + response, err := kaytu.Ec2InstanceWastageRequest(request, job.processor.kaytuAcccessToken) + if err != nil { + return err + } + + job.item = ComputeInstanceItem{ + ProjectId: job.item.ProjectId, + Name: job.item.Name, + Id: job.item.Id, + MachineType: job.item.MachineType, + Region: job.item.Region, + Platform: job.item.Platform, + OptimizationLoading: false, + Preferences: job.item.Preferences, + Skipped: false, + LazyLoadingEnabled: false, + SkipReason: "NA", + Metrics: job.item.Metrics, + Wastage: *response, + } + + job.processor.items.Set(job.item.Id, job.item) + job.processor.publishOptimizationItem(job.item.ToOptimizationItem()) + + return nil +} diff --git a/plugin/processor/interface.go b/plugin/processor/interface.go index 9b0a71e..e53220b 100644 --- a/plugin/processor/interface.go +++ b/plugin/processor/interface.go @@ -4,4 +4,5 @@ import "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" type PluginProcessor interface { ReEvaluate(id string, items []*golang.PreferenceItem) + ExportNonInteractive() *golang.NonInteractiveExport } diff --git a/plugin/service.go b/plugin/service.go index bf49ea2..5644795 100644 --- a/plugin/service.go +++ b/plugin/service.go @@ -1,7 +1,9 @@ package plugin import ( + "context" "fmt" + "log" "github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang" "github.com/kaytu-io/kaytu/pkg/plugin/sdk" @@ -13,7 +15,7 @@ import ( ) type GCPPlugin struct { - stream golang.Plugin_RegisterClient + stream *sdk.StreamController processor processor.PluginProcessor } @@ -46,33 +48,80 @@ func (p *GCPPlugin) GetConfig() golang.RegisterConfig { Columns: []*golang.ChartColumnItem{ { - Id: "instance_name", - Name: "Instance Name", - Width: uint32(10), - Sortable: true, + Id: "resource_id", + Name: "Resource ID", + Width: uint32(10), + }, + { + Id: "resource_name", + Name: "Resource Name", + Width: uint32(10), + }, + { + Id: "region", + Name: "Region", + Width: uint32(15), + }, + { + Id: "platform", + Name: "Platform", + Width: uint32(15), + }, + { + Id: "total_saving", + Name: "Total Saving (Monthly)", + Width: uint32(40), + }, + { + Id: "x_kaytu_right_arrow", + Name: "", + Width: uint32(1), }, }, }, DevicesChart: &golang.ChartDefinition{ Columns: []*golang.ChartColumnItem{ { - Id: "instance_name", - Name: "Instance Name", - Width: uint32(10), - Sortable: true, + Id: "resource_id", + Name: "Resource ID", + Width: uint32(10), + }, + { + Id: "resource_name", + Name: "Resource Name", + Width: uint32(10), + }, + { + Id: "resource_type", + Name: "Resource Type", + Width: uint32(10), + }, + { + Id: "project_id", + Name: "Project ID", + Width: uint32(10), }, { - Id: "project_id", - Name: "Project ID", - Width: uint32(10), - Sortable: true, + Id: "current_cost", + Name: "Current Cost", + Width: uint32(20), + }, + { + Id: "right_sized_cost", + Name: "Right sized Cost", + Width: 20, + }, + { + Id: "savings", + Name: "Savings", + Width: 20, }, }, }, } } -func (p *GCPPlugin) SetStream(stream golang.Plugin_RegisterClient) { +func (p *GCPPlugin) SetStream(stream *sdk.StreamController) { p.stream = stream } @@ -86,6 +135,24 @@ func (p *GCPPlugin) StartProcess(cmd string, flags map[string]string, kaytuAcces }, ) + metricClient := gcp.NewCloudMonitoring( + []string{ + "https://www.googleapis.com/auth/monitoring.read", + }, + ) + + log.Println("Initializing clients") + + err := gcpProvider.InitializeClient(context.Background()) + if err != nil { + return err + } + + err = metricClient.InitializeClient(context.Background()) + if err != nil { + return err + } + publishOptimizationItem := func(item *golang.ChartOptimizationItem) { p.stream.Send(&golang.PluginMessage{ PluginMessage: &golang.PluginMessage_Coi{ @@ -117,6 +184,7 @@ func (p *GCPPlugin) StartProcess(cmd string, flags map[string]string, kaytuAcces if cmd == "compute-instance" { p.processor = compute_instance.NewComputeInstanceProcessor( gcpProvider, + metricClient, publishOptimizationItem, publishResultSummary, kaytuAccessToken, @@ -125,7 +193,7 @@ func (p *GCPPlugin) StartProcess(cmd string, flags map[string]string, kaytuAcces } else { return fmt.Errorf("invalid command: %s", cmd) } - jobQueue.SetOnFinish(func() { + jobQueue.SetOnFinish(func(ctx context.Context) { publishResultsReady(true) }) @@ -135,3 +203,7 @@ func (p *GCPPlugin) StartProcess(cmd string, flags map[string]string, kaytuAcces func (p *GCPPlugin) ReEvaluate(evaluate *golang.ReEvaluate) { p.processor.ReEvaluate(evaluate.Id, evaluate.Preferences) } + +func (p *GCPPlugin) ExportNonInteractive() *golang.NonInteractiveExport { + return nil +} diff --git a/utils/extract_string.go b/utils/extract_string.go new file mode 100644 index 0000000..1315b58 --- /dev/null +++ b/utils/extract_string.go @@ -0,0 +1,11 @@ +package util + +import "strings" + +func TrimmedString(s string, sep string) string { + + slc := strings.Split(s, sep) + + return slc[len(slc)-1] + +}