Skip to content

Commit

Permalink
Merge pull request #32 from kaytu-io/compute-instance-recommendations
Browse files Browse the repository at this point in the history 
Compute instance recommendations
  • Loading branch information
@artaasadi
artaasadi committed Jul 1, 2024
2 parents ed9a22d + a9110c7 commit 96282fc
Show file tree
Hide file tree
Showing 10 changed files with 450 additions and 394 deletions.
17 changes: 9 additions & 8 deletions plugin/gcp/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ package gcp
import (
"context"
"fmt"
"github.com/kaytu-io/plugin-gcp/plugin/kaytu"
golang2 "github.com/kaytu-io/plugin-gcp/plugin/proto/src/golang"
"google.golang.org/api/iterator"
"google.golang.org/protobuf/types/known/wrapperspb"

monitoring "cloud.google.com/go/monitoring/apiv3/v2"
"cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
Expand Down Expand Up @@ -59,8 +60,8 @@ func (c *CloudMonitoring) NewTimeSeriesRequest(

}

func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest) ([]kaytu.Datapoint, error) {
var dps []kaytu.Datapoint
func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest) ([]*golang2.DataPoint, error) {
var dps []*golang2.DataPoint

it := c.client.ListTimeSeries(context.Background(), request)
for {
Expand All @@ -80,13 +81,13 @@ func (c *CloudMonitoring) GetMetric(request *monitoringpb.ListTimeSeriesRequest)

}

func convertDatapoints(resp *monitoringpb.TimeSeries) []kaytu.Datapoint {
var dps []kaytu.Datapoint
func convertDatapoints(resp *monitoringpb.TimeSeries) []*golang2.DataPoint {
var dps []*golang2.DataPoint
for _, dp := range resp.GetPoints() {
dps = append(dps, kaytu.Datapoint{
dps = append(dps, &golang2.DataPoint{
Value: dp.GetValue().GetDoubleValue(),
StartTime: dp.GetInterval().GetStartTime().AsTime(),
EndTime: dp.GetInterval().GetEndTime().AsTime(),
StartTime: wrapperspb.Int64(dp.GetInterval().GetStartTime().GetSeconds()),
EndTime: wrapperspb.Int64(dp.GetInterval().GetEndTime().GetSeconds()),
})
}
return dps
Expand Down
13 changes: 8 additions & 5 deletions plugin/processor/compute_instance/compute_instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@ package compute_instance

import (
"fmt"
"github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang"
"github.com/kaytu-io/kaytu/pkg/plugin/sdk"
"github.com/kaytu-io/kaytu/pkg/style"
"github.com/kaytu-io/kaytu/pkg/utils"
"github.com/kaytu-io/plugin-gcp/plugin/gcp"
golang2 "github.com/kaytu-io/plugin-gcp/plugin/proto/src/golang"
util "github.com/kaytu-io/plugin-gcp/utils"
"strconv"
"strings"
"sync/atomic"

"github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang"
"github.com/kaytu-io/kaytu/pkg/plugin/sdk"
"github.com/kaytu-io/plugin-gcp/plugin/gcp"
util "github.com/kaytu-io/plugin-gcp/utils"
)

type ComputeInstanceProcessor struct {
Expand All @@ -23,6 +23,7 @@ type ComputeInstanceProcessor struct {
kaytuAcccessToken string
jobQueue *sdk.JobQueue
lazyloadCounter atomic.Uint32
client golang2.OptimizationClient

summary util.ConcurrentMap[string, ComputeInstanceSummary]
}
Expand All @@ -34,6 +35,7 @@ func NewComputeInstanceProcessor(
publishResultSummary func(summary *golang.ResultSummary),
kaytuAcccessToken string,
jobQueue *sdk.JobQueue,
client golang2.OptimizationClient,
) *ComputeInstanceProcessor {
r := &ComputeInstanceProcessor{
provider: prv,
Expand All @@ -44,6 +46,7 @@ func NewComputeInstanceProcessor(
kaytuAcccessToken: kaytuAcccessToken,
jobQueue: jobQueue,
lazyloadCounter: atomic.Uint32{},
client: client,
}

jobQueue.Push(NewListComputeInstancesJob(r))
Expand Down
80 changes: 44 additions & 36 deletions plugin/processor/compute_instance/compute_instance_item.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"fmt"
"github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang"
"github.com/kaytu-io/kaytu/pkg/utils"
"github.com/kaytu-io/plugin-gcp/plugin/kaytu"
golang2 "github.com/kaytu-io/plugin-gcp/plugin/proto/src/golang"
"google.golang.org/api/compute/v1"
"google.golang.org/protobuf/types/known/wrapperspb"
"maps"
Expand All @@ -26,9 +26,9 @@ type ComputeInstanceItem struct {
SkipReason string
Instance *computepb.Instance
Disks []compute.Disk
Metrics map[string][]kaytu.Datapoint
DisksMetrics map[string]map[string][]kaytu.Datapoint
Wastage kaytu.GcpComputeInstanceWastageResponse
Metrics map[string][]*golang2.DataPoint
DisksMetrics map[string]map[string][]*golang2.DataPoint
Wastage golang2.GCPComputeOptimizationResponse
}

func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[string]*golang.Properties) {
Expand All @@ -49,55 +49,55 @@ func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[stri
}

row.Values["current_cost"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost),
Value: utils.FormatPriceFloat(i.Wastage.Rightsizing.Current.Cost),
}

RegionProperty := &golang.Property{
Key: "Region",
Current: i.Wastage.RightSizing.Current.Region,
Current: i.Wastage.Rightsizing.Current.Region,
}

MachineTypeProperty := &golang.Property{
Key: "Machine Type",
Current: i.Wastage.RightSizing.Current.MachineType,
Current: i.Wastage.Rightsizing.Current.MachineType,
}
MachineFamilyProperty := &golang.Property{
Key: "Machine Family",
Current: i.Wastage.RightSizing.Current.MachineFamily,
Current: i.Wastage.Rightsizing.Current.MachineFamily,
}
CPUProperty := &golang.Property{
Key: " CPU",
Current: fmt.Sprintf("%d", i.Wastage.RightSizing.Current.CPU),
Average: utils.Percentage(i.Wastage.RightSizing.CPU.Avg),
Max: utils.Percentage(i.Wastage.RightSizing.CPU.Max),
Current: fmt.Sprintf("%d", i.Wastage.Rightsizing.Current.Cpu),
Average: utils.Percentage(PWrapperDouble(i.Wastage.Rightsizing.Cpu.Avg)),
Max: utils.Percentage(PWrapperDouble(i.Wastage.Rightsizing.Cpu.Max)),
}

memoryProperty := &golang.Property{
Key: " MemoryMB",
Current: fmt.Sprintf("%d MB", i.Wastage.RightSizing.Current.MemoryMb),
Average: utils.Percentage(i.Wastage.RightSizing.Memory.Avg),
Max: utils.Percentage(i.Wastage.RightSizing.Memory.Max),
Current: fmt.Sprintf("%d MB", i.Wastage.Rightsizing.Current.MemoryMb),
Average: utils.Percentage(PWrapperDouble(i.Wastage.Rightsizing.Memory.Avg)),
Max: utils.Percentage(PWrapperDouble(i.Wastage.Rightsizing.Memory.Max)),
}

row.Values["project_id"] = &golang.ChartRowItem{
Value: i.ProjectId,
}

row.Values["current_cost"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost),
Value: utils.FormatPriceFloat(i.Wastage.Rightsizing.Current.Cost),
}

if i.Wastage.RightSizing.Recommended != nil {
if i.Wastage.Rightsizing.Recommended != nil {
row.Values["right_sized_cost"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Recommended.Cost),
Value: utils.FormatPriceFloat(i.Wastage.Rightsizing.Recommended.Cost),
}
row.Values["savings"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost - i.Wastage.RightSizing.Recommended.Cost),
Value: utils.FormatPriceFloat(i.Wastage.Rightsizing.Current.Cost - i.Wastage.Rightsizing.Recommended.Cost),
}
RegionProperty.Recommended = i.Wastage.RightSizing.Recommended.Region
MachineTypeProperty.Recommended = i.Wastage.RightSizing.Recommended.MachineType
CPUProperty.Recommended = fmt.Sprintf("%d", i.Wastage.RightSizing.Recommended.CPU)
memoryProperty.Recommended = fmt.Sprintf("%d MB", i.Wastage.RightSizing.Recommended.MemoryMb)
RegionProperty.Recommended = i.Wastage.Rightsizing.Recommended.Region
MachineTypeProperty.Recommended = i.Wastage.Rightsizing.Recommended.MachineType
CPUProperty.Recommended = fmt.Sprintf("%d", i.Wastage.Rightsizing.Recommended.Cpu)
memoryProperty.Recommended = fmt.Sprintf("%d MB", i.Wastage.Rightsizing.Recommended.MemoryMb)
}

props := make(map[string]*golang.Properties)
Expand All @@ -123,7 +123,7 @@ func (i ComputeInstanceItem) ComputeDiskDevice() ([]*golang.ChartRow, map[string

for _, d := range i.Disks {
key := strconv.FormatUint(d.Id, 10)
disk := i.Wastage.VolumeRightSizing[key]
disk := i.Wastage.VolumesRightsizing[key]

row := golang.ChartRow{
RowId: key,
Expand Down Expand Up @@ -164,26 +164,26 @@ func (i ComputeInstanceItem) ComputeDiskDevice() ([]*golang.ChartRow, map[string
DiskReadIopsProperty := &golang.Property{
Key: " Read IOPS Expectation",
Current: fmt.Sprintf("%d", disk.Current.ReadIopsLimit),
Average: utils.PFloat64ToString(disk.ReadIops.Avg),
Max: utils.PFloat64ToString(disk.ReadIops.Max),
Average: utils.PFloat64ToString(PWrapperDouble(disk.ReadIops.Avg)),
Max: utils.PFloat64ToString(PWrapperDouble(disk.ReadIops.Max)),
}
DiskWriteIopsProperty := &golang.Property{
Key: " Write IOPS Expectation",
Current: fmt.Sprintf("%d", disk.Current.WriteIopsLimit),
Average: utils.PFloat64ToString(disk.WriteIops.Avg),
Max: utils.PFloat64ToString(disk.WriteIops.Max),
Average: utils.PFloat64ToString(PWrapperDouble(disk.WriteIops.Avg)),
Max: utils.PFloat64ToString(PWrapperDouble(disk.WriteIops.Max)),
}
DiskReadThroughputProperty := &golang.Property{
Key: " Read Throughput Expectation",
Current: fmt.Sprintf("%.2f Mb", disk.Current.ReadThroughputLimit),
Average: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.ReadThroughput.Avg)),
Max: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.ReadThroughput.Max)),
Average: fmt.Sprintf("%s Mb", utils.PFloat64ToString(PWrapperDouble(disk.ReadThroughput.Avg))),
Max: fmt.Sprintf("%s Mb", utils.PFloat64ToString(PWrapperDouble(disk.ReadThroughput.Max))),
}
DiskWriteThroughputProperty := &golang.Property{
Key: " Write Throughput Expectation",
Current: fmt.Sprintf("%.2f Mb", disk.Current.WriteThroughputLimit),
Average: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.WriteThroughput.Avg)),
Max: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.WriteThroughput.Max)),
Average: fmt.Sprintf("%s Mb", utils.PFloat64ToString(PWrapperDouble(disk.WriteThroughput.Avg))),
Max: fmt.Sprintf("%s Mb", utils.PFloat64ToString(PWrapperDouble(disk.WriteThroughput.Max))),
}

if disk.Recommended != nil {
Expand Down Expand Up @@ -252,12 +252,12 @@ func (i ComputeInstanceItem) ToOptimizationItem() *golang.ChartOptimizationItem
status = "press enter to load"
} else if i.OptimizationLoading {
status = "loading"
} else if i.Wastage.RightSizing.Recommended != nil {
} else if i.Wastage.Rightsizing.Recommended != nil {
totalSaving := 0.0
totalCurrentCost := 0.0
totalSaving += i.Wastage.RightSizing.Current.Cost - i.Wastage.RightSizing.Recommended.Cost
totalCurrentCost += i.Wastage.RightSizing.Current.Cost
for _, d := range i.Wastage.VolumeRightSizing {
totalSaving += i.Wastage.Rightsizing.Current.Cost - i.Wastage.Rightsizing.Recommended.Cost
totalCurrentCost += i.Wastage.Rightsizing.Current.Cost
for _, d := range i.Wastage.VolumesRightsizing {
totalSaving += d.Current.Cost - d.Recommended.Cost
totalCurrentCost += d.Current.Cost
}
Expand Down Expand Up @@ -296,7 +296,7 @@ func (i ComputeInstanceItem) ToOptimizationItem() *golang.ChartOptimizationItem
DevicesChartRows: deviceRows,
DevicesProperties: deviceProps,
Preferences: i.Preferences,
Description: i.Wastage.RightSizing.Description,
Description: i.Wastage.Rightsizing.Description,
Loading: i.OptimizationLoading,
Skipped: i.Skipped,
SkipReason: wrapperspb.String(i.SkipReason),
Expand All @@ -305,3 +305,11 @@ func (i ComputeInstanceItem) ToOptimizationItem() *golang.ChartOptimizationItem

return coi
}

func PWrapperDouble(v *wrapperspb.DoubleValue) *float64 {
if v == nil {
return nil
}
value := v.GetValue()
return &value
}
9 changes: 4 additions & 5 deletions plugin/processor/compute_instance/job_get_compute_instance_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,12 @@ package compute_instance
import (
"context"
"fmt"
golang2 "github.com/kaytu-io/plugin-gcp/plugin/proto/src/golang"
"google.golang.org/api/compute/v1"
"log"
"strconv"
"time"

"github.com/kaytu-io/plugin-gcp/plugin/kaytu"

"cloud.google.com/go/compute/apiv1/computepb"
"cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
"github.com/kaytu-io/plugin-gcp/plugin/preferences"
Expand Down Expand Up @@ -92,10 +91,10 @@ func (job *GetComputeInstanceMetricsJob) Run(ctx context.Context) error {
return err
}

disksMetrics := make(map[string]map[string][]kaytu.Datapoint)
disksMetrics := make(map[string]map[string][]*golang2.DataPoint)
for _, disk := range job.disks {
id := strconv.FormatUint(disk.Id, 10)
disksMetrics[id] = make(map[string][]kaytu.Datapoint)
disksMetrics[id] = make(map[string][]*golang2.DataPoint)

diskReadIopsRequest := job.processor.metricProvider.NewTimeSeriesRequest(
fmt.Sprintf(
Expand Down Expand Up @@ -194,7 +193,7 @@ func (job *GetComputeInstanceMetricsJob) Run(ctx context.Context) error {
disksMetrics[id]["DiskWriteThroughput"] = diskWriteThroughputMetrics
}

instanceMetrics := make(map[string][]kaytu.Datapoint)
instanceMetrics := make(map[string][]*golang2.DataPoint)

instanceMetrics["cpuUtilization"] = cpumetric
instanceMetrics["memoryUtilization"] = memoryMetric
Expand Down

0 comments on commit 96282fc

Please sign in to comment.