Merge pull request #11 from kaytu-io/compute-instance-recommendations

Compute instance recommendations

.github/workflows/main.yaml

@@ -46,26 +46,26 @@ jobs:
         with:
           name: new_tag
           path: new_tag.txt
-  release:
-    runs-on: ubuntu-latest
-    needs:
-      - tag
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.GH_TOKEN }}
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v5
-        with:
-          distribution: goreleaser
-          version: latest
-          args: release --clean
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+#  release:
+#    runs-on: ubuntu-latest
+#    needs:
+#      - tag
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v4
+#        with:
+#          fetch-depth: 0
+#          token: ${{ secrets.GH_TOKEN }}
+#      - name: Set up Go
+#        uses: actions/setup-go@v4
+#        with:
+#          go-version: stable
+#      - name: Run GoReleaser
+#        uses: goreleaser/goreleaser-action@v5
+#        with:
+#          distribution: goreleaser
+#          version: latest
+#          args: release --clean
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 

go.mod

@@ -5,17 +5,15 @@ go 1.21.5
 require (
 	cloud.google.com/go/compute v1.25.1
 	cloud.google.com/go/monitoring v1.18.0
-	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.6
 	github.com/google/uuid v1.6.0
-	github.com/kaytu-io/kaytu v0.10.5
+	github.com/kaytu-io/kaytu v0.14.0-rc.0.0.20240619114201-51f5733d10b6
 	golang.org/x/oauth2 v0.21.0
 	google.golang.org/api v0.169.0
 	google.golang.org/protobuf v1.34.2
 )
 
 require (
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
-	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

go.sum

@@ -8,10 +8,6 @@ cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1h
 cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4=
 cloud.google.com/go/monitoring v1.18.0/go.mod h1:c92vVBCeq/OB4Ioyo+NbN2U7tlg5ZH41PZcdvfc+Lcg=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.6 h1:UVjxYe8VGpwXYcmBcciBHlQrNssdEvntXCPWmnRR15U=
-github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.6/go.mod h1:4V6VDA0kZavRn71+sLpVna75oobnlG+gwtnNcBwZhu4=
-github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
-github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -65,8 +61,8 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh
 github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/kaytu-io/kaytu v0.10.5 h1:EGrvEQTjnIC06ix2rUt0RBlxHjS50pHKifSVQJOH1Ho=
-github.com/kaytu-io/kaytu v0.10.5/go.mod h1:BhFDEm+6ckgusHsrDCdgSH+b/wMhuRsq29L8/PJe2Sk=
+github.com/kaytu-io/kaytu v0.14.0-rc.0.0.20240619114201-51f5733d10b6 h1:OyrvbIttya/35K+6N1tyI953BlKclIkHGWIZ/IPYheg=
+github.com/kaytu-io/kaytu v0.14.0-rc.0.0.20240619114201-51f5733d10b6/go.mod h1:csDLaCgTnp/D9SZuo55KjOzp/BYfkhkeSkGuEiCljtc=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

plugin/gcp/compute.go

@@ -13,7 +13,8 @@ import (
 )
 
 type Compute struct {
-	client *compute.InstancesClient
+	instancesClient   *compute.InstancesClient
+	machineTypeClient *compute.MachineTypesClient
 	GCP
 }
 
@@ -38,15 +39,28 @@ func (c *Compute) InitializeClient(ctx context.Context) error {
 		return err
 	}
 
+	machineTypeClient, err := compute.NewMachineTypesRESTClient(
+		ctx,
+		option.WithCredentials(c.GCP.credentials),
+	)
+	if err != nil {
+		return err
+	}
+
 	// log.Println(instancesClient)
 
-	c.client = instancesClient
+	c.instancesClient = instancesClient
+	c.machineTypeClient = machineTypeClient
 
 	return nil
 }
 
 func (c *Compute) CloseClient() error {
-	err := c.client.Close()
+	err := c.instancesClient.Close()
+	if err != nil {
+		return err
+	}
+	err = c.machineTypeClient.Close()
 	if err != nil {
 		return err
 	}
@@ -59,7 +73,7 @@ func (c *Compute) ListAllInstances() error {
 		Project: c.ProjectID,
 	}
 
-	it := c.client.AggregatedList(context.Background(), req)
+	it := c.instancesClient.AggregatedList(context.Background(), req)
 
 	log.Println("instances found: ")
 
@@ -92,7 +106,7 @@ func (c *Compute) GetAllInstances() ([]*computepb.Instance, error) {
 		Project: c.ProjectID,
 	}
 
-	it := c.client.AggregatedList(context.Background(), req)
+	it := c.instancesClient.AggregatedList(context.Background(), req)
 
 	log.Println("instances found: ")
 
@@ -121,3 +135,22 @@ func (c *Compute) GetAllInstances() ([]*computepb.Instance, error) {
 	}
 	return allInstances, nil
 }
+
+func (c *Compute) GetMemory(InstanceMachineType string, zone string) (*int32, error) {
+
+	request := &computepb.GetMachineTypeRequest{
+		Project:     c.ProjectID,
+		MachineType: InstanceMachineType,
+		Zone:        zone,
+	}
+
+	machineType, err := c.machineTypeClient.Get(context.Background(), request)
+	if err != nil {
+		return nil, err
+	}
+
+	memory := machineType.GetMemoryMb()
+
+	return &memory, nil
+
+}

plugin/gcp/compute_test.go

@@ -3,6 +3,7 @@ package gcp
 import (
 	"context"
 	"log"
+	"os"
 	"testing"
 )
 
@@ -59,3 +60,29 @@ func TestGetAllInstances(t *testing.T) {
 	compute.CloseClient()
 
 }
+
+// TEST_INSTANCE_ZONE="us-east1-b" TEST_INSTANCE_MACHINE_TYPE="e2-micro" TEST_INSTANCE_ID="7828543314219019363" make testgcp
+func TestGetMemory(t *testing.T) {
+
+	machineType := os.Getenv("TEST_INSTANCE_MACHINE_TYPE")
+	zone := os.Getenv("TEST_INSTANCE_ZONE")
+
+	log.Printf("running %s", t.Name())
+	compute := NewCompute(
+		[]string{
+			"https://www.googleapis.com/auth/compute.readonly",
+		},
+	)
+	err := compute.InitializeClient(context.Background())
+	if err != nil {
+		t.Errorf("[%s]: %s", t.Name(), err.Error())
+	}
+
+	memory, err := compute.GetMemory(machineType, zone)
+	if err != nil {
+		t.Errorf("[%s]: %s", t.Name(), err.Error())
+	}
+
+	log.Printf("Memory : %d", &memory)
+
+}

plugin/gcp/config.go

@@ -21,17 +21,28 @@ func NewGCP(scopes []string) GCP {
 }
 
 func (g *GCP) GetCredentials(ctx context.Context) error {
-	credentials, err := google.FindDefaultCredentials(
+	var err error
+
+	g.credentials, err = google.FindDefaultCredentials(
 		ctx,
 		g.Scopes...,
 	)
 	if err != nil {
 		return err
 	}
 
-	json.Unmarshal(credentials.JSON, g) //this will store project id from credentials
+	g.ProjectID = g.credentials.ProjectID
+
+	json.Unmarshal(g.credentials.JSON, g) //this will store project id from credentials
 
-	g.credentials = credentials
-	// log.Println(g.ProjectID)
 	return nil
 }
+
+func (g *GCP) Identify() map[string]string {
+
+	identification := map[string]string{
+		"project_id": g.ProjectID,
+	}
+
+	return identification
+}

plugin/gcp/config_test.go

@@ -0,0 +1,22 @@
+package gcp
+
+import (
+	"log"
+	"testing"
+)
+
+func TestIdentify(t *testing.T) {
+
+	test_project_id := "test-project-id"
+	gcp := GCP{
+		ProjectID: test_project_id,
+	}
+
+	identification := gcp.Identify()
+	log.Println(identification)
+
+	if identification["project_id"] != test_project_id {
+		t.Error("TestIdentify failed")
+	}
+
+}

plugin/gcp/metrics.go

@@ -3,12 +3,11 @@ package gcp
 import (
 	"context"
 	"fmt"
-	"time"
+	"github.com/kaytu-io/plugin-gcp/plugin/kaytu"
+	"google.golang.org/api/iterator"
 
 	monitoring "cloud.google.com/go/monitoring/apiv3/v2"
 	"cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
-	"google.golang.org/protobuf/types/known/durationpb"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 type CloudMonitoring struct {
@@ -44,46 +43,51 @@ func (c *CloudMonitoring) CloseClient() error {
 	return nil
 }
 
-func (c *CloudMonitoring) NewInstanceMetricRequest(
-	metricName string, // fully qualified name of the metric
-	instanceID string, // compute instance ID
-	startTime time.Time, // start time of requested time series
-	endTime time.Time, // end time of requested time series
-	periodInSeconds int64, // period, for which the datapoints will be aggregated into one, in seconds
+func (c *CloudMonitoring) NewTimeSeriesRequest(
+	filter string, // filter for time series metric, containing metric name and resource label
+	interval *monitoringpb.TimeInterval, // interval containing start and end time of the requested time series
+	aggregation *monitoringpb.Aggregation, // operations to perform on time series data before returning
 ) *monitoringpb.ListTimeSeriesRequest {
 
-	request := &monitoringpb.ListTimeSeriesRequest{
-		Name: fmt.Sprintf("projects/%s", c.ProjectID),
-		Filter: fmt.Sprintf(
-			`metric.type="%s" AND resource.labels.instance_id="%s"`,
-			metricName,
-			instanceID,
-		),
-		Interval: &monitoringpb.TimeInterval{
-			EndTime:   timestamppb.New(endTime),
-			StartTime: timestamppb.New(startTime),
-		},
-		Aggregation: &monitoringpb.Aggregation{
-			AlignmentPeriod: &durationpb.Duration{
-				Seconds: periodInSeconds,
-			},
-			PerSeriesAligner: monitoringpb.Aggregation_ALIGN_MEAN, // will represent all the datapoints in the above period, with a mean
-		},
-		View: monitoringpb.ListTimeSeriesRequest_FULL,
+	return &monitoringpb.ListTimeSeriesRequest{
+		Name:        fmt.Sprintf("projects/%s", c.ProjectID),
+		Filter:      filter,
+		Interval:    interval,
+		Aggregation: aggregation,
+		View:        monitoringpb.ListTimeSeriesRequest_FULL,
 	}
 
-	return request
 }
 
-func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest) *monitoringpb.TimeSeries {
+func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest) ([]kaytu.Datapoint, error) {
+	var dps []kaytu.Datapoint
 
 	it := c.client.ListTimeSeries(context.Background(), request)
-
-	resp, err := it.Next()
-	if err != nil {
-		panic(err)
+	for {
+		resp, err := it.Next()
+
+		if err != nil {
+			if err == iterator.Done {
+				break
+			} else {
+				return nil, err
+			}
+		}
+		dps = append(dps, convertDatapoints(resp)...)
 	}
 
-	return resp
+	return dps, nil
 
 }
+
+func convertDatapoints(resp *monitoringpb.TimeSeries) []kaytu.Datapoint {
+	var dps []kaytu.Datapoint
+	for _, dp := range resp.GetPoints() {
+		dps = append(dps, kaytu.Datapoint{
+			Value:     dp.GetValue().GetDoubleValue(),
+			StartTime: dp.GetInterval().GetStartTime().AsTime(),
+			EndTime:   dp.GetInterval().GetEndTime().AsTime(),
+		})
+	}
+	return dps
+}