Skip to content

Commit

Permalink
Merge pull request #25 from kaytu-io/compute-instance-recommendations
Browse files Browse the repository at this point in the history 
Compute instance recommendations
  • Loading branch information
@artaasadi
artaasadi committed Jun 25, 2024
2 parents c658e81 + 43fe926 commit d0123fa
Show file tree
Hide file tree
Showing 6 changed files with 168 additions and 30 deletions.
34 changes: 26 additions & 8 deletions plugin/gcp/compute.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,19 @@
package gcp

import (
"context"
"log"

compute "cloud.google.com/go/compute/apiv1"
computeApi "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/compute/apiv1/computepb"
"context"
"google.golang.org/api/compute/v1"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"log"
)

type Compute struct {
instancesClient *compute.InstancesClient
machineTypeClient *compute.MachineTypesClient
instancesClient *computeApi.InstancesClient
machineTypeClient *computeApi.MachineTypesClient
computeService *compute.Service
GCP
}

Expand All @@ -31,15 +32,23 @@ func (c *Compute) InitializeClient(ctx context.Context) error {
// log.Println(string(c.GCP.credentials.JSON))
// log.Println(c.GCP.ProjectID)

instancesClient, err := compute.NewInstancesRESTClient(
instancesClient, err := computeApi.NewInstancesRESTClient(
ctx,
option.WithCredentials(c.GCP.credentials),
)
if err != nil {
return err
}

machineTypeClient, err := compute.NewMachineTypesRESTClient(
machineTypeClient, err := computeApi.NewMachineTypesRESTClient(
ctx,
option.WithCredentials(c.GCP.credentials),
)
if err != nil {
return err
}

computeService, err := compute.NewService(
ctx,
option.WithCredentials(c.GCP.credentials),
)
Expand All @@ -51,6 +60,7 @@ func (c *Compute) InitializeClient(ctx context.Context) error {

c.instancesClient = instancesClient
c.machineTypeClient = machineTypeClient
c.computeService = computeService

return nil
}
Expand Down Expand Up @@ -136,6 +146,14 @@ func (c *Compute) GetAllInstances() ([]*computepb.Instance, error) {
return allInstances, nil
}

func (c *Compute) GetDiskDetails(zone, diskName string) (*compute.Disk, error) {
disk, err := c.computeService.Disks.Get(c.ProjectID, zone, diskName).Context(context.Background()).Do()
if err != nil {
return nil, err
}
return disk, nil
}

func (c *Compute) GetMemory(InstanceMachineType string, zone string) (*int32, error) {

request := &computepb.GetMachineTypeRequest{
Expand Down
35 changes: 18 additions & 17 deletions plugin/kaytu/compute_instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@ type GcpComputeInstance struct {
}

type GcpComputeDisk struct {
HashedDiskId string `json:"hashedDiskId"`
Zone string `json:"zone"`
Region string `json:"region"`
DiskType string `json:"diskType"`
DiskSize *int32 `json:"diskSize"`
HashedDiskId string `json:"hashedDiskId"`
Zone string `json:"zone"`
Region string `json:"region"`
DiskType string `json:"diskType"`
DiskSize *int64 `json:"diskSize"`
ProvisionedIops *int64 `json:"provisionedIops"`
}

type RightsizingGcpComputeInstance struct {
Expand All @@ -38,31 +39,31 @@ type GcpComputeInstanceRightsizingRecommendation struct {
}

type GcpComputeInstanceWastageRequest struct {
RequestId *string `json:"requestId"`
CliVersion *string `json:"cliVersion"`
Identification map[string]string `json:"identification"`
Instance GcpComputeInstance `json:"instance"`
Disks []GcpComputeDisk `json:"disks"`
Metrics map[string][]Datapoint `json:"metrics"`
DiskMetrics map[string]map[string][]Datapoint `json:"diskMetrics"`
Region string `json:"region"`
Preferences map[string]*string `json:"preferences"`
Loading bool `json:"loading"`
RequestId *string `json:"requestId"`
CliVersion *string `json:"cliVersion"`
Identification map[string]string `json:"identification"`
Instance GcpComputeInstance `json:"instance"`
Disks []GcpComputeDisk `json:"disks"`
Metrics map[string][]Datapoint `json:"metrics"`
DiskCapacityUsed map[string]float64 `json:"diskCapacityUsed"`
Region string `json:"region"`
Preferences map[string]*string `json:"preferences"`
Loading bool `json:"loading"`
}

type RightsizingGcpComputeDisk struct {
Zone string `json:"zone"`
Region string `json:"region"`
DiskType string `json:"diskType"`
DiskSize *int32 `json:"diskSize"`
DiskSize *int64 `json:"diskSize"`
Cost float64 `json:"cost"`
}

type GcpComputeDiskRecommendation struct {
Current RightsizingGcpComputeDisk
Recommended *RightsizingGcpComputeDisk

Size Usage `json:"size"`
UsedCapacity float64 `json:"usedCapacity"`

Description string `json:"description"`
}
Expand Down
82 changes: 82 additions & 0 deletions plugin/processor/compute_instance/compute_instance_item.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package compute_instance

import (
"fmt"
"google.golang.org/api/compute/v1"
"maps"
"strconv"

"github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang"
"github.com/kaytu-io/kaytu/pkg/utils"
Expand All @@ -22,6 +24,7 @@ type ComputeInstanceItem struct {
Skipped bool
LazyLoadingEnabled bool
SkipReason string
Disks []compute.Disk
Metrics map[string][]kaytu.Datapoint
Wastage kaytu.GcpComputeInstanceWastageResponse
}
Expand Down Expand Up @@ -110,15 +113,94 @@ func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[stri
return &row, props
}

func (i ComputeInstanceItem) ComputeDiskDevice() ([]*golang.ChartRow, map[string]*golang.Properties) {
var rows []*golang.ChartRow
props := make(map[string]*golang.Properties)

for _, d := range i.Disks {
key := strconv.FormatUint(d.Id, 10)
disk := i.Wastage.VolumeRightSizing[key]

row := golang.ChartRow{
RowId: key,
Values: make(map[string]*golang.ChartRowItem),
}
row.RowId = key

row.Values["project_id"] = &golang.ChartRowItem{
Value: i.ProjectId,
}
row.Values["resource_id"] = &golang.ChartRowItem{
Value: key,
}
row.Values["resource_name"] = &golang.ChartRowItem{
Value: d.Name,
}
row.Values["resource_type"] = &golang.ChartRowItem{
Value: "Compute Disk",
}

row.Values["current_cost"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(disk.Current.Cost),
}

RegionProperty := &golang.Property{
Key: "Region",
Current: disk.Current.Region,
}

DiskTypeProperty := &golang.Property{
Key: "Disk Type",
Current: disk.Current.DiskType,
}
DiskSizeProperty := &golang.Property{
Key: "Disk Size",
Current: fmt.Sprintf("%d GB", d.SizeGb),
}

if disk.Recommended != nil {
row.Values["right_sized_cost"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(disk.Recommended.Cost),
}
row.Values["savings"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(disk.Current.Cost - disk.Recommended.Cost),
}
RegionProperty.Recommended = disk.Recommended.Region
DiskTypeProperty.Recommended = disk.Recommended.DiskType
if disk.Recommended.DiskSize != nil {
DiskSizeProperty.Recommended = fmt.Sprintf("%d GB", *disk.Recommended.DiskSize)
}
}

properties := &golang.Properties{}

properties.Properties = append(properties.Properties, RegionProperty)
properties.Properties = append(properties.Properties, DiskTypeProperty)
properties.Properties = append(properties.Properties, DiskSizeProperty)

props[key] = properties
rows = append(rows, &row)
}

return rows, props
}

func (i ComputeInstanceItem) Devices() ([]*golang.ChartRow, map[string]*golang.Properties) {

var deviceRows []*golang.ChartRow
deviceProps := make(map[string]*golang.Properties)

instanceRows, instanceProps := i.ComputeInstanceDevice()
diskRows, diskProps := i.ComputeDiskDevice()

fmt.Println("==========")
fmt.Println("disks", diskRows)
fmt.Println("instance", *instanceRows)

deviceRows = append(deviceRows, instanceRows)
deviceRows = append(deviceRows, diskRows...)
maps.Copy(deviceProps, instanceProps)
maps.Copy(deviceProps, diskProps)

return deviceRows, deviceProps
}
Expand Down
23 changes: 19 additions & 4 deletions plugin/processor/compute_instance/job_compute_instance_list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package compute_instance

import (
"context"
"google.golang.org/api/compute/v1"
"log"
"strconv"
"strings"

"github.com/kaytu-io/plugin-gcp/plugin/preferences"
util "github.com/kaytu-io/plugin-gcp/utils"
Expand Down Expand Up @@ -40,6 +42,21 @@ func (job *ListComputeInstancesJob) Run(ctx context.Context) error {
log.Printf("# of instances: %d", len(instances))

for _, instance := range instances {
var disks []compute.Disk
for _, attachedDisk := range instance.Disks {
diskURLParts := strings.Split(*attachedDisk.Source, "/")
diskName := diskURLParts[len(diskURLParts)-1]

zoneURLParts := strings.Split(*instance.Zone, "/")
instanceZone := zoneURLParts[len(zoneURLParts)-1]

diskDetails, err := job.processor.provider.GetDiskDetails(instanceZone, diskName)
if err != nil {
return err
}
disks = append(disks, *diskDetails)
}

oi := ComputeInstanceItem{
ProjectId: job.processor.provider.ProjectID,
Name: *instance.Name,
Expand All @@ -52,18 +69,16 @@ func (job *ListComputeInstancesJob) Run(ctx context.Context) error {
Skipped: false,
LazyLoadingEnabled: false,
SkipReason: "NA",
Disks: disks,
Metrics: nil,
}

log.Printf("OI instance: %s", oi.Name)

job.processor.items.Set(oi.Id, oi)
job.processor.publishOptimizationItem(oi.ToOptimizationItem())
}

for _, instance := range instances {

job.processor.jobQueue.Push(NewGetComputeInstanceMetricsJob(job.processor, instance))
job.processor.jobQueue.Push(NewGetComputeInstanceMetricsJob(job.processor, instance, disks))
}

return nil
Expand Down
6 changes: 5 additions & 1 deletion plugin/processor/compute_instance/job_get_compute_instance_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package compute_instance
import (
"context"
"fmt"
"google.golang.org/api/compute/v1"
"log"
"strconv"
"time"
Expand All @@ -20,12 +21,14 @@ import (
type GetComputeInstanceMetricsJob struct {
processor *ComputeInstanceProcessor
instance *computepb.Instance
disks []compute.Disk
}

func NewGetComputeInstanceMetricsJob(processor *ComputeInstanceProcessor, instance *computepb.Instance) *GetComputeInstanceMetricsJob {
func NewGetComputeInstanceMetricsJob(processor *ComputeInstanceProcessor, instance *computepb.Instance, disks []compute.Disk) *GetComputeInstanceMetricsJob {
return &GetComputeInstanceMetricsJob{
processor: processor,
instance: instance,
disks: disks,
}
}

Expand Down Expand Up @@ -111,6 +114,7 @@ func (job *GetComputeInstanceMetricsJob) Run(ctx context.Context) error {
Skipped: false,
LazyLoadingEnabled: false,
SkipReason: "NA",
Disks: job.disks,
Metrics: instanceMetrics,
}

Expand Down
18 changes: 18 additions & 0 deletions plugin/processor/compute_instance/job_optimize_compute_instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package compute_instance
import (
"context"
"fmt"
"strconv"

"github.com/google/uuid"
"github.com/kaytu-io/kaytu/pkg/utils"
Expand Down Expand Up @@ -36,6 +37,21 @@ func (job *OptimizeComputeInstancesJob) Run(ctx context.Context) error {

requestId := uuid.NewString()

var disks []kaytu.GcpComputeDisk
diskFilled := make(map[string]float64)
for _, disk := range job.item.Disks {
id := strconv.FormatUint(disk.Id, 10)
disks = append(disks, kaytu.GcpComputeDisk{
HashedDiskId: id,
DiskSize: &disk.SizeGb,
DiskType: disk.Type,
Region: disk.Region,
ProvisionedIops: &disk.ProvisionedIops,
Zone: disk.Zone,
})
diskFilled[id] = 0
}

request := kaytu.GcpComputeInstanceWastageRequest{
RequestId: &requestId,
CliVersion: &version.VERSION,
Expand All @@ -45,6 +61,7 @@ func (job *OptimizeComputeInstancesJob) Run(ctx context.Context) error {
Zone: job.item.Region,
MachineType: job.item.MachineType,
},
Disks: disks,
Metrics: job.item.Metrics,
Region: job.item.Region,
Preferences: preferences.Export(job.item.Preferences),
Expand All @@ -69,6 +86,7 @@ func (job *OptimizeComputeInstancesJob) Run(ctx context.Context) error {
LazyLoadingEnabled: false,
SkipReason: "NA",
Metrics: job.item.Metrics,
Disks: job.item.Disks,
Wastage: *response,
}

Expand Down

0 comments on commit d0123fa

Please sign in to comment.