Get disk pricing working (#118)

This implements a `ListDisks` method in `gke` that is then used to list out disks for the GKE projects by zone.
`ListDisks` will currently return a raw `compute.Disk` struct that is filtered by
2. Disks only associated with GKE clusters

- Relates to #95

docs/metrics/gcp/gke.md

@@ -1,6 +1,53 @@
 # GKE Compute Metrics
 
-| Metric name                                            | Metric type | Description                                                   | Labels                                                                                                                                                                                                                                                                                                                                                                                                                           |
-|--------------------------------------------------------|-------------|---------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Metric name                                                | Metric type | Description                                                                                 | Labels                                                                                                                                                                                                                                                                                                                                                                                                                           |
+|------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | cloudcost_gcp_gke_instance_cpu_usd_per_core_hour           | Gauge       | The processing cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(core*h) | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `instance`=&lt;name of the compute instance&gt; <br/> `region`=&lt;GCP region code&gt; <br/> `family`=&lt;broader compute family (n1, n2, c3 ...) &gt; <br/> `machine_type`=&lt;specific machine type, e.g.: n2-standard-2&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `price_tier`=&lt;spot\|ondemand&gt; |
 | cloudcost_gcp_gke_compute_instance_memory_usd_per_gib_hour | Gauge       | The memory cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(GiB*h)      | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `instance`=&lt;name of the compute instance&gt; <br/> `region`=&lt;GCP region code&gt; <br/> `family`=&lt;broader compute family (n1, n2, c3 ...) &gt; <br/> `machine_type`=&lt;specific machine type, e.g.: n2-standard-2&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `price_tier`=&lt;spot\|ondemand&gt; |
+| cloudcost_gcp_gke_persistent_volume_usd_per_gib_hour       | Gauge       | The cost of a GKE Persistent Volume in USD/(GiB*h)                                          | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `namespace`=&lt;The namespace the pvc was created for&gt; <br/> `persistentvolume`=&lt;Name of the persistent volume&gt; <br/> `region`=&lt;The region the pvc was created in&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `storage_class`=&lt;pd-standard\|pd-ssd\|pd-balanced\|pd-extreme&gt;             |
+
+## Persistent Volumes
+
+There's two sources of data for persistent volumes:
+- Skus from the Billing API
+- Disk metadata from compute API
+
+There's a bit of a disconnect between the two.
+Sku's descriptions have the following format:
+```
+Balanced PD Capacity in <Region>
+Commitment v1: Local SSD In <Region>
+Extreme PD Capacity in <Region>
+Extreme PD IOPS in <Region>
+Hyperdisk Balanced Capacity in <Region>
+Hyperdisk Balanced IOPS in <Region>
+Hyperdisk Balanced Throughput in <Region>
+Hyperdisk Extreme Capacity in <Region>
+Hyperdisk Extreme IOPS in <Region>
+Hyperdisk Throughput Capacity in <Region>
+Hyperdisk Throughput Throughput Capacity in <Region>
+Regional Balanced PD Capacity in <Region>
+Regional SSD backed PD Capacity in <Region>
+Regional Storage PD Capacity in <Region>
+SSD backed Local Storage attached to Spot Preemptible VM in <Region>
+SSD backed Local Storage in <Region
+SSD backed PD Capacity in <Region>
+Storage PD Capacity in <Region>
+```
+
+Generically, the sku descriptions have the following format:
+```
+<sku-type> PD Capacity in <Region>
+```
+
+Disk metadata has the following format:
+```
+projects/<project>/zones/<zone>/disks/<disk-type>
+```
+
+To map the sku to the disk type, we can use the following mapping:
+
+- Storage PD Capacity -> pd-standard
+- SSD backed PD Capacity -> pd-ssd
+- Balanced PD Capacity -> pd-balanced
+- Extreme PD Capacity -> pd-extreme

pkg/aws/s3/s3.go

@@ -17,13 +17,13 @@ import (
 	cloudcost_exporter "github.com/grafana/cloudcost-exporter"
 	"github.com/grafana/cloudcost-exporter/pkg/aws/costexplorer"
 	"github.com/grafana/cloudcost-exporter/pkg/provider"
+	"github.com/grafana/cloudcost-exporter/pkg/utils"
 )
 
 // HoursInMonth is the average hours in a month, used to calculate the cost of storage
 // If we wanted to be clever, we can get the number of hours in the current month
 // 365.25 * 24 / 12 ~= 730.5
 const (
-	HoursInMonth = 730.5
 	// This needs to line up with yace so we can properly join the data in PromQL
 	StandardLabel = "StandardStorage"
 	subsystem     = "aws_s3"
@@ -421,7 +421,7 @@ func unitCostForComponent(component string, pricing *Pricing) float64 {
 	case "Requests-Tier1", "Requests-Tier2":
 		return pricing.Cost / (pricing.Usage / 1000)
 	case "TimedStorage":
-		return (pricing.Cost / HoursInMonth) / pricing.Usage
+		return (pricing.Cost / utils.HoursInMonth) / pricing.Usage
 	default:
 		return pricing.Cost / pricing.Usage
 	}

pkg/google/billing/billing.go

@@ -3,7 +3,7 @@ package billing
 import (
 	"context"
 	"errors"
-	"strings"
+	"log"
 
 	billingv1 "cloud.google.com/go/billing/apiv1"
 	"cloud.google.com/go/billing/apiv1/billingpb"
@@ -41,13 +41,9 @@ func GetPricing(ctx context.Context, billingService *billingv1.CloudCatalogClien
 			if errors.Is(err, iterator.Done) {
 				break
 			}
-			// keep going if we get an error
-		}
-
-		// We don't include licensing skus in our pricing map
-		if !strings.Contains(strings.ToLower(sku.Description), "licensing") {
-			skus = append(skus, sku)
+			log.Println(err) // keep going if we get an error
 		}
+		skus = append(skus, sku)
 	}
 	return skus
 }

pkg/google/billing/billing_test_helpers.go

@@ -177,6 +177,27 @@ func (s *FakeCloudCatalogServer) ListSkus(_ context.Context, req *billingpb.List
 					},
 				},
 			},
+			{
+				Name:           "standard-storage",
+				Description:    "Storage PD Capacity",
+				ServiceRegions: []string{"us-central1"},
+				Category: &billingpb.Category{
+					ResourceFamily: "Storage",
+				},
+				PricingInfo: []*billingpb.PricingInfo{{
+					PricingExpression: &billingpb.PricingExpression{
+						TieredRates: []*billingpb.PricingExpression_TierRate{{
+							UnitPrice: &money.Money{
+								Nanos: 0.0,
+							},
+						}, {
+							UnitPrice: &money.Money{
+								Nanos: 1e9,
+							},
+						}},
+					},
+				}},
+			},
 		},
 	}, nil
 }
@@ -218,6 +239,27 @@ func (s *FakeCloudCatalogServerSlimResults) ListSkus(_ context.Context, req *bil
 					},
 				},
 			},
+			{
+				Name:           "standard-storage",
+				Description:    "Storage PD Capacity",
+				ServiceRegions: []string{"us-central1"},
+				Category: &billingpb.Category{
+					ResourceFamily: "Storage",
+				},
+				PricingInfo: []*billingpb.PricingInfo{{
+					PricingExpression: &billingpb.PricingExpression{
+						TieredRates: []*billingpb.PricingExpression_TierRate{{
+							UnitPrice: &money.Money{
+								Nanos: 0.0,
+							},
+						}, {
+							UnitPrice: &money.Money{
+								Nanos: 1e9,
+							},
+						}},
+					},
+				}},
+			},
 		},
 	}, nil
 }

pkg/google/compute/machinespec.go

@@ -11,6 +11,7 @@ import (
 var (
 	re              = regexp.MustCompile(`\bin\b`)
 	GkeClusterLabel = "goog-k8s-cluster-name"
+	GkeRegionLabel  = "goog-k8s-cluster-location"
 )
 
 // MachineSpec is a slimmed down representation of a google compute.Instance struct