From 82143750bd5d9895d608396b4497776d26ea230e Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Wed, 8 May 2024 10:16:33 -0400 Subject: [PATCH] AWS network costs - Add data transfer direction to daily summary - Separate node network costs out into network unattributed --- ...rting_ocpawscostlineitem_daily_summary.sql | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql index f974a6fa93..c3fe82541b 100644 --- a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS {{schema | sqlsafe}}.aws_openshift_daily_resource_mat region varchar, unit varchar, usage_amount double, + data_transfer_direction varchar, currency_code varchar, unblended_cost double, blended_cost double, @@ -172,6 +173,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp region, unit, usage_amount, + data_transfer_direction, currency_code, unblended_cost, blended_cost, @@ -198,6 +200,19 @@ SELECT cast(uuid() as varchar) as uuid, nullif(aws.product_region, '') as region, max(nullif(aws.pricing_unit, '')) as unit, sum(aws.lineitem_usageamount) as usage_amount, + -- Determine network direction + CASE + -- Is this a network record? + WHEN aws.lineitem_productcode = 'AmazonEC2' AND aws.product_productfamily = 'Data Transfer' THEN + -- Yes, it's a network. What's the direction? + CASE + WHEN strpos(aws.lineitem_usagetype, 'In-Bytes') > 0 THEN 'IN' + WHEN strpos(aws.lineitem_usagetype, 'Out-Bytes') > 0 THEN 'OUT' + WHEN (strpos(aws.lineitem_usagetype, 'Regional-Bytes') > 0 AND strpos(lineitem_operation, '-In') > 0) THEN 'IN' + WHEN (strpos(aws.lineitem_usagetype, 'Regional-Bytes') > 0 AND strpos(lineitem_operation, '-Out') > 0)THEN 'OUT' + ELSE NULL + END + END as data_transfer_direction, max(nullif(aws.lineitem_currencycode, '')) as currency_code, sum(aws.lineitem_unblendedcost) as unblended_cost, sum(aws.lineitem_blendedcost) as blended_cost, @@ -446,6 +461,8 @@ SELECT aws.uuid as aws_uuid, AND aws.ocp_source = {{ocp_source_uuid}} AND aws.year = {{year}} AND aws.month = {{month}} + -- Filter out Node Networks Costs since they cannot be attributed to a namespace and are account for later + AND aws.data_transfer_direction IS NULL GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels ; @@ -567,6 +584,7 @@ SELECT aws.uuid as aws_uuid, ) AND namespace != 'Worker unallocated' AND namespace != 'Platform unallocated' + AND namespace != 'Network unattributed' WHERE ocp.source = {{ocp_source_uuid}} AND ocp.year = {{year}} AND lpad(ocp.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters @@ -600,6 +618,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily region, unit, usage_amount, + data_transfer_direction, currency_code, unblended_cost, markup_cost, @@ -648,6 +667,7 @@ SELECT pds.aws_uuid, region, unit, usage_amount / aws_uuid_count as usage_amount, + NULL AS data_transfer_direction, currency_code, CASE WHEN resource_id_matched = TRUE AND data_source = 'Pod' THEN ({{pod_column | sqlsafe}} / nullif({{node_column | sqlsafe}}, 0)) * unblended_cost @@ -717,6 +737,15 @@ LEFT JOIN postgres.{{schema | sqlsafe}}.reporting_awsaccountalias AS aa WHERE pds.ocp_source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} ; +-- Put Node Network Costs back in +INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( + +) +FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws + ON aws.usage_start = ocp.usage_start + AND strpos(aws.resource_id, ocp.resource_id) != 0 + INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p ( uuid, report_period_id,