Skip to content

Commit

Permalink
Merge pull request #28 from kaytu-io/compute-instance-recommendations
Browse files Browse the repository at this point in the history 
fix: add iops and throughput metrics to disks
  • Loading branch information
@artaasadi
artaasadi committed Jun 30, 2024
2 parents d0123fa + 458e876 commit 2c6b366
Show file tree
Hide file tree
Showing 9 changed files with 361 additions and 79 deletions.
40 changes: 24 additions & 16 deletions plugin/kaytu/compute_instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,31 +39,39 @@ type GcpComputeInstanceRightsizingRecommendation struct {
}

type GcpComputeInstanceWastageRequest struct {
RequestId *string `json:"requestId"`
CliVersion *string `json:"cliVersion"`
Identification map[string]string `json:"identification"`
Instance GcpComputeInstance `json:"instance"`
Disks []GcpComputeDisk `json:"disks"`
Metrics map[string][]Datapoint `json:"metrics"`
DiskCapacityUsed map[string]float64 `json:"diskCapacityUsed"`
Region string `json:"region"`
Preferences map[string]*string `json:"preferences"`
Loading bool `json:"loading"`
RequestId *string `json:"requestId"`
CliVersion *string `json:"cliVersion"`
Identification map[string]string `json:"identification"`
Instance GcpComputeInstance `json:"instance"`
Disks []GcpComputeDisk `json:"disks"`
Metrics map[string][]Datapoint `json:"metrics"`
DisksMetrics map[string]map[string][]Datapoint `json:"diskMetrics"`
Region string `json:"region"`
Preferences map[string]*string `json:"preferences"`
Loading bool `json:"loading"`
}

type RightsizingGcpComputeDisk struct {
Zone string `json:"zone"`
Region string `json:"region"`
DiskType string `json:"diskType"`
DiskSize *int64 `json:"diskSize"`
Cost float64 `json:"cost"`
Zone string `json:"zone"`
Region string `json:"region"`
DiskType string `json:"diskType"`
DiskSize int64 `json:"diskSize"`
ReadIopsLimit int64 `json:"readIopsLimit"`
WriteIopsLimit int64 `json:"writeIopsLimit"`
ReadThroughputLimit float64 `json:"readThroughputLimit"`
WriteThroughputLimit float64 `json:"writeThroughputLimit"`

Cost float64 `json:"cost"`
}

type GcpComputeDiskRecommendation struct {
Current RightsizingGcpComputeDisk
Recommended *RightsizingGcpComputeDisk

UsedCapacity float64 `json:"usedCapacity"`
ReadIops Usage `json:"readIops"`
WriteIops Usage `json:"writeIops"`
ReadThroughput Usage `json:"readThroughput"`
WriteThroughput Usage `json:"writeThroughput"`

Description string `json:"description"`
}
Expand Down
4 changes: 4 additions & 0 deletions plugin/preferences/default.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,8 @@ var DefaultComputeEnginePreferences = []*golang.PreferenceItem{
{Service: "ComputeInstance", Key: "MemoryGB", Alias: "Memory", IsNumber: true, Unit: "GiB"},
{Service: "ComputeInstance", Key: "CPUBreathingRoom", IsNumber: true, Value: wrapperspb.String("10"), PreventPinning: true, Unit: "%"},
{Service: "ComputeInstance", Key: "MemoryBreathingRoom", IsNumber: true, Value: wrapperspb.String("10"), PreventPinning: true, Unit: "%"},
{Service: "ComputeInstance", Key: "ExcludeUpsizingFeature", Value: wrapperspb.String("Yes"), PreventPinning: true, PossibleValues: []string{"No", "Yes"}},

{Service: "ComputeDisk", Key: "DiskType"},
{Service: "ComputeDisk", Key: "DiskSizeGb", IsNumber: true, Unit: "GiB"},
}
109 changes: 98 additions & 11 deletions plugin/processor/compute_instance/compute_instance.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package compute_instance

import (
"log"
"fmt"
"github.com/kaytu-io/kaytu/pkg/utils"
"strconv"
"strings"
"sync/atomic"

"github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang"
"github.com/kaytu-io/kaytu/pkg/plugin/sdk"
Expand All @@ -17,6 +21,7 @@ type ComputeInstanceProcessor struct {
publishResultSummary func(summary *golang.ResultSummary)
kaytuAcccessToken string
jobQueue *sdk.JobQueue
lazyloadCounter atomic.Uint32
}

func NewComputeInstanceProcessor(
Expand All @@ -27,7 +32,6 @@ func NewComputeInstanceProcessor(
kaytuAcccessToken string,
jobQueue *sdk.JobQueue,
) *ComputeInstanceProcessor {
log.Println("creating processor")
r := &ComputeInstanceProcessor{
provider: prv,
metricProvider: metricPrv,
Expand All @@ -36,22 +40,105 @@ func NewComputeInstanceProcessor(
publishResultSummary: publishResultSummary,
kaytuAcccessToken: kaytuAcccessToken,
jobQueue: jobQueue,
// configuration: configurations,
// lazyloadCounter: lazyloadCounter,
lazyloadCounter: atomic.Uint32{},
}

jobQueue.Push(NewListComputeInstancesJob(r))
return r
}

func (m *ComputeInstanceProcessor) ReEvaluate(id string, items []*golang.PreferenceItem) {
log.Println("Reevaluate unimplemented")
// v, _ := m.items.Get(id)
// v.Preferences = items
// m.items.Set(id, v)
// m.jobQueue.Push(NewOptimizeEC2InstanceJob(m, v))
v, _ := m.items.Get(id)
v.Preferences = items
m.items.Set(id, v)
fmt.Println("HERE===================")
fmt.Println("Instance Metrics", len(v.Metrics))
fmt.Println("Disk Metrics", len(v.DisksMetrics))
fmt.Println("Disks", len(v.Disks))
v.OptimizationLoading = true
m.publishOptimizationItem(v.ToOptimizationItem())
m.jobQueue.Push(NewOptimizeComputeInstancesJob(m, v))
}

func (p *ComputeInstanceProcessor) ExportNonInteractive() *golang.NonInteractiveExport {
return nil
func (m *ComputeInstanceProcessor) ExportNonInteractive() *golang.NonInteractiveExport {
return &golang.NonInteractiveExport{
Csv: m.exportCsv(),
}
}

func (m *ComputeInstanceProcessor) exportCsv() []*golang.CSVRow {
headers := []string{
"Project ID", "Region", "Resource Type", "Resource ID", "Resource Name", "Platform",
"Device Runtime (Hrs)", "Current Cost", "Recommendation Cost", "Net Savings",
"Current Spec", "Suggested Spec", "Parent Device", "Justification", "Additional Details",
}
var rows []*golang.CSVRow
rows = append(rows, &golang.CSVRow{Row: headers})

m.items.Range(func(key string, value ComputeInstanceItem) bool {
var additionalDetails []string
var rightSizingCost, saving, recSpec string
if value.Wastage.RightSizing.Recommended != nil {
rightSizingCost = utils.FormatPriceFloat(value.Wastage.RightSizing.Recommended.Cost)
saving = utils.FormatPriceFloat(value.Wastage.RightSizing.Current.Cost - value.Wastage.RightSizing.Recommended.Cost)
recSpec = value.Wastage.RightSizing.Recommended.MachineType

additionalDetails = append(additionalDetails,
fmt.Sprintf("Machine Type:: Current: %s - Recommended: %s", value.Wastage.RightSizing.Current.MachineType,
value.Wastage.RightSizing.Recommended.MachineType))
additionalDetails = append(additionalDetails,
fmt.Sprintf("Region:: Current: %s - Recommended: %s", value.Wastage.RightSizing.Current.Region,
value.Wastage.RightSizing.Recommended.Region))
additionalDetails = append(additionalDetails,
fmt.Sprintf("CPU:: Current: %d - Recommended: %d", value.Wastage.RightSizing.Current.CPU,
value.Wastage.RightSizing.Recommended.CPU))
additionalDetails = append(additionalDetails,
fmt.Sprintf("Memory:: Current: %d - Recommended: %d", value.Wastage.RightSizing.Current.MemoryMb,
value.Wastage.RightSizing.Recommended.MemoryMb))
}
computeRow := []string{
value.ProjectId, value.Region, "Compute Instance", value.Id, value.Name, value.Platform,
"730 Hrs", utils.FormatPriceFloat(value.Wastage.RightSizing.Current.Cost), rightSizingCost, saving,
value.Wastage.RightSizing.Current.MachineType, recSpec, "None", value.Wastage.RightSizing.Description, strings.Join(additionalDetails, "---")}

rows = append(rows, &golang.CSVRow{Row: computeRow})

for _, d := range value.Disks {
dKey := strconv.FormatUint(d.Id, 10)
disk := value.Wastage.VolumeRightSizing[dKey]
var diskAdditionalDetails []string
var diskRightSizingCost, diskSaving, diskRecSpec string
if disk.Recommended != nil {
diskRightSizingCost = utils.FormatPriceFloat(disk.Recommended.Cost)
diskSaving = utils.FormatPriceFloat(disk.Current.Cost - disk.Recommended.Cost)
diskRecSpec = fmt.Sprintf("%s / %d GB", disk.Recommended.DiskType, disk.Recommended.DiskSize)

diskAdditionalDetails = append(diskAdditionalDetails,
fmt.Sprintf("Region:: Current: %s - Recommended: %s", disk.Current.Region,
disk.Recommended.Region))
diskAdditionalDetails = append(diskAdditionalDetails,
fmt.Sprintf("ReadIopsExpectation:: Current: %d - Recommended: %d", disk.Current.ReadIopsLimit,
disk.Recommended.ReadIopsLimit))
diskAdditionalDetails = append(diskAdditionalDetails,
fmt.Sprintf("WriteIopsExpectation:: Current: %d - Recommended: %d", disk.Current.WriteIopsLimit,
disk.Recommended.WriteIopsLimit))
diskAdditionalDetails = append(diskAdditionalDetails,
fmt.Sprintf("ReadThroughputExpectation:: Current: %.2f - Recommended: %.2f", disk.Current.ReadThroughputLimit,
disk.Recommended.ReadThroughputLimit))
diskAdditionalDetails = append(diskAdditionalDetails,
fmt.Sprintf("WriteThroughputExpectation:: Current: %.2f - Recommended: %.2f", disk.Current.WriteThroughputLimit,
disk.Recommended.WriteThroughputLimit))
}
diskRow := []string{
value.ProjectId, value.Region, "Compute Disk", dKey, d.Name, "N/A",
"730 Hrs", utils.FormatPriceFloat(disk.Current.Cost), diskRightSizingCost, diskSaving,
fmt.Sprintf("%s / %d GB", disk.Current.DiskType, disk.Current.DiskSize), diskRecSpec,
"None", value.Wastage.RightSizing.Description, strings.Join(diskAdditionalDetails, "---")}

rows = append(rows, &golang.CSVRow{Row: diskRow})
}

return true
})
return rows
}
84 changes: 65 additions & 19 deletions plugin/processor/compute_instance/compute_instance_item.go
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
package compute_instance

import (
"cloud.google.com/go/compute/apiv1/computepb"
"fmt"
"google.golang.org/api/compute/v1"
"maps"
"strconv"

"github.com/kaytu-io/kaytu/pkg/plugin/proto/src/golang"
"github.com/kaytu-io/kaytu/pkg/utils"
"github.com/kaytu-io/plugin-gcp/plugin/kaytu"
"google.golang.org/api/compute/v1"
"google.golang.org/protobuf/types/known/wrapperspb"
"maps"
"strconv"
)

type ComputeInstanceItem struct {
Expand All @@ -24,8 +24,10 @@ type ComputeInstanceItem struct {
Skipped bool
LazyLoadingEnabled bool
SkipReason string
Instance *computepb.Instance
Disks []compute.Disk
Metrics map[string][]kaytu.Datapoint
DisksMetrics map[string]map[string][]kaytu.Datapoint
Wastage kaytu.GcpComputeInstanceWastageResponse
}

Expand All @@ -50,9 +52,9 @@ func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[stri
Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost),
}

ZoneProperty := &golang.Property{
Key: "Zone",
Current: i.Wastage.RightSizing.Current.Zone,
RegionProperty := &golang.Property{
Key: "Region",
Current: i.Wastage.RightSizing.Current.Region,
}

MachineTypeProperty := &golang.Property{
Expand All @@ -66,6 +68,8 @@ func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[stri
CPUProperty := &golang.Property{
Key: " CPU",
Current: fmt.Sprintf("%d", i.Wastage.RightSizing.Current.CPU),
Average: utils.Percentage(i.Wastage.RightSizing.CPU.Avg),
Max: utils.Percentage(i.Wastage.RightSizing.CPU.Max),
}

memoryProperty := &golang.Property{
Expand All @@ -90,7 +94,7 @@ func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[stri
row.Values["savings"] = &golang.ChartRowItem{
Value: utils.FormatPriceFloat(i.Wastage.RightSizing.Current.Cost - i.Wastage.RightSizing.Recommended.Cost),
}
ZoneProperty.Recommended = i.Wastage.RightSizing.Recommended.Zone
RegionProperty.Recommended = i.Wastage.RightSizing.Recommended.Region
MachineTypeProperty.Recommended = i.Wastage.RightSizing.Recommended.MachineType
CPUProperty.Recommended = fmt.Sprintf("%d", i.Wastage.RightSizing.Recommended.CPU)
memoryProperty.Recommended = fmt.Sprintf("%d MB", i.Wastage.RightSizing.Recommended.MemoryMb)
Expand All @@ -99,7 +103,7 @@ func (i ComputeInstanceItem) ComputeInstanceDevice() (*golang.ChartRow, map[stri
props := make(map[string]*golang.Properties)
properties := &golang.Properties{}

properties.Properties = append(properties.Properties, ZoneProperty)
properties.Properties = append(properties.Properties, RegionProperty)
properties.Properties = append(properties.Properties, MachineTypeProperty)
properties.Properties = append(properties.Properties, MachineFamilyProperty)
properties.Properties = append(properties.Properties, &golang.Property{
Expand Down Expand Up @@ -155,7 +159,31 @@ func (i ComputeInstanceItem) ComputeDiskDevice() ([]*golang.ChartRow, map[string
}
DiskSizeProperty := &golang.Property{
Key: "Disk Size",
Current: fmt.Sprintf("%d GB", d.SizeGb),
Current: fmt.Sprintf("%d GB", disk.Current.DiskSize),
}
DiskReadIopsProperty := &golang.Property{
Key: " Read IOPS Expectation",
Current: fmt.Sprintf("%d", disk.Current.ReadIopsLimit),
Average: utils.PFloat64ToString(disk.ReadIops.Avg),
Max: utils.PFloat64ToString(disk.ReadIops.Max),
}
DiskWriteIopsProperty := &golang.Property{
Key: " Write IOPS Expectation",
Current: fmt.Sprintf("%d", disk.Current.WriteIopsLimit),
Average: utils.PFloat64ToString(disk.WriteIops.Avg),
Max: utils.PFloat64ToString(disk.WriteIops.Max),
}
DiskReadThroughputProperty := &golang.Property{
Key: " Read Throughput Expectation",
Current: fmt.Sprintf("%.2f Mb", disk.Current.ReadThroughputLimit),
Average: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.ReadThroughput.Avg)),
Max: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.ReadThroughput.Max)),
}
DiskWriteThroughputProperty := &golang.Property{
Key: " Write Throughput Expectation",
Current: fmt.Sprintf("%.2f Mb", disk.Current.WriteThroughputLimit),
Average: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.WriteThroughput.Avg)),
Max: fmt.Sprintf("%s Mb", utils.PFloat64ToString(disk.WriteThroughput.Max)),
}

if disk.Recommended != nil {
Expand All @@ -167,16 +195,28 @@ func (i ComputeInstanceItem) ComputeDiskDevice() ([]*golang.ChartRow, map[string
}
RegionProperty.Recommended = disk.Recommended.Region
DiskTypeProperty.Recommended = disk.Recommended.DiskType
if disk.Recommended.DiskSize != nil {
DiskSizeProperty.Recommended = fmt.Sprintf("%d GB", *disk.Recommended.DiskSize)
}
DiskReadIopsProperty.Recommended = fmt.Sprintf("%d", disk.Recommended.ReadIopsLimit)
DiskWriteIopsProperty.Recommended = fmt.Sprintf("%d", disk.Recommended.WriteIopsLimit)
DiskReadThroughputProperty.Recommended = fmt.Sprintf("%.2f Mb", disk.Recommended.ReadThroughputLimit)
DiskWriteThroughputProperty.Recommended = fmt.Sprintf("%.2f Mb", disk.Recommended.WriteThroughputLimit)
DiskSizeProperty.Recommended = fmt.Sprintf("%d GB", disk.Recommended.DiskSize)
}

properties := &golang.Properties{}

properties.Properties = append(properties.Properties, RegionProperty)
properties.Properties = append(properties.Properties, DiskTypeProperty)
properties.Properties = append(properties.Properties, DiskSizeProperty)
properties.Properties = append(properties.Properties, &golang.Property{
Key: "IOPS",
})
properties.Properties = append(properties.Properties, DiskReadIopsProperty)
properties.Properties = append(properties.Properties, DiskWriteIopsProperty)
properties.Properties = append(properties.Properties, &golang.Property{
Key: "Throughput",
})
properties.Properties = append(properties.Properties, DiskReadThroughputProperty)
properties.Properties = append(properties.Properties, DiskWriteThroughputProperty)

props[key] = properties
rows = append(rows, &row)
Expand All @@ -193,10 +233,6 @@ func (i ComputeInstanceItem) Devices() ([]*golang.ChartRow, map[string]*golang.P
instanceRows, instanceProps := i.ComputeInstanceDevice()
diskRows, diskProps := i.ComputeDiskDevice()

fmt.Println("==========")
fmt.Println("disks", diskRows)
fmt.Println("instance", *instanceRows)

deviceRows = append(deviceRows, instanceRows)
deviceRows = append(deviceRows, diskRows...)
maps.Copy(deviceProps, instanceProps)
Expand All @@ -210,11 +246,21 @@ func (i ComputeInstanceItem) ToOptimizationItem() *golang.ChartOptimizationItem
deviceRows, deviceProps := i.Devices()

status := ""
if i.Wastage.RightSizing.Recommended != nil {
if i.Skipped {
status = fmt.Sprintf("skipped - %s", i.SkipReason)
} else if i.LazyLoadingEnabled && !i.OptimizationLoading {
status = "press enter to load"
} else if i.OptimizationLoading {
status = "loading"
} else if i.Wastage.RightSizing.Recommended != nil {
totalSaving := 0.0
totalCurrentCost := 0.0
totalSaving += i.Wastage.RightSizing.Current.Cost - i.Wastage.RightSizing.Recommended.Cost
totalCurrentCost += i.Wastage.RightSizing.Current.Cost
for _, d := range i.Wastage.VolumeRightSizing {
totalSaving += d.Current.Cost - d.Recommended.Cost
totalCurrentCost += d.Current.Cost
}
status = fmt.Sprintf("%s (%.2f%%)", utils.FormatPriceFloat(totalSaving), (totalSaving/totalCurrentCost)*100)
}

Expand Down Expand Up @@ -250,7 +296,7 @@ func (i ComputeInstanceItem) ToOptimizationItem() *golang.ChartOptimizationItem
DevicesChartRows: deviceRows,
DevicesProperties: deviceProps,
Preferences: i.Preferences,
Description: "description placeholder",
Description: i.Wastage.RightSizing.Description,
Loading: i.OptimizationLoading,
Skipped: i.Skipped,
SkipReason: wrapperspb.String(i.SkipReason),
Expand Down

0 comments on commit 2c6b366

Please sign in to comment.