From 1e5836a8e6d7a384af1da5dbfe0b533075cc6d2e Mon Sep 17 00:00:00 2001 From: yingyingqiqi <48437786+yingyingqiqi@users.noreply.github.com> Date: Mon, 14 Aug 2023 16:36:18 +0000 Subject: [PATCH 1/3] Add credit consumption to cost per query --- models/cost_per_query.sql | 9 ++++++++- models/cost_per_query.yml | 6 ++++++ models/query_history_enriched.sql | 3 +++ models/query_history_enriched.yml | 6 ++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/models/cost_per_query.sql b/models/cost_per_query.sql index eec7622..07a8954 100644 --- a/models/cost_per_query.sql +++ b/models/cost_per_query.sql @@ -73,7 +73,8 @@ query_cost as ( select query_seconds_per_hour.*, credits_billed_hourly.credits_used_compute * daily_rates.effective_rate as actual_warehouse_cost, - credits_billed_hourly.credits_used_compute * query_seconds_per_hour.fraction_of_total_query_time_in_hour * daily_rates.effective_rate as allocated_compute_cost_in_hour + credits_billed_hourly.credits_used_compute * query_seconds_per_hour.fraction_of_total_query_time_in_hour * daily_rates.effective_rate as allocated_compute_cost_in_hour, + credits_billed_hourly.credits_used_compute * query_seconds_per_hour.fraction_of_total_query_time_in_hour as allocated_compute_credits_in_hour from query_seconds_per_hour inner join credits_billed_hourly on query_seconds_per_hour.warehouse_id = credits_billed_hourly.warehouse_id @@ -91,6 +92,7 @@ cost_per_query as ( any_value(end_time) as end_time, any_value(execution_start_time) as execution_start_time, sum(allocated_compute_cost_in_hour) as compute_cost, + sum(allocated_compute_credits_in_hour) as compute_credits, any_value(credits_used_cloud_services) as credits_used_cloud_services, any_value(ran_on_warehouse) as ran_on_warehouse from query_cost @@ -114,6 +116,7 @@ all_queries as ( end_time, execution_start_time, compute_cost, + compute_credits, credits_used_cloud_services, ran_on_warehouse from cost_per_query @@ -126,6 +129,7 @@ all_queries as ( end_time, execution_start_time, 0 as compute_cost, + 0 as compute_credits, credits_used_cloud_services, ran_on_warehouse from filtered_queries @@ -139,12 +143,15 @@ select all_queries.end_time, all_queries.execution_start_time, all_queries.compute_cost, + all_queries.compute_credits, -- For the most recent day, which is not yet complete, this calculation won't be perfect. -- For example, at 12PM on the latest day, it's possible that cloud credits make up <10% of compute cost, so the queries -- from that day are not allocated any cloud_services_cost. The next time the model runs, after we have the full day of data, -- this may change if cloud credits make up >10% of compute cost. (div0(all_queries.credits_used_cloud_services, credits_billed_daily.daily_credits_used_cloud_services) * credits_billed_daily.daily_billable_cloud_services) * coalesce(daily_rates.effective_rate, current_rates.effective_rate) as cloud_services_cost, + div0(all_queries.credits_used_cloud_services, credits_billed_daily.daily_credits_used_cloud_services) * credits_billed_daily.daily_billable_cloud_services as cloud_services_credits, all_queries.compute_cost + cloud_services_cost as query_cost, + all_queries.compute_credits + cloud_services_credits as query_credits, all_queries.ran_on_warehouse, coalesce(daily_rates.currency, current_rates.currency) as currency from all_queries diff --git a/models/cost_per_query.yml b/models/cost_per_query.yml index ed94bb4..62f18ba 100644 --- a/models/cost_per_query.yml +++ b/models/cost_per_query.yml @@ -18,10 +18,16 @@ models: description: When the query began executing on the warehouse (in the UTC time zone). This will always be after the start_time. - name: compute_cost description: Compute costs associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. + - name: compute_credits + description: Compute credits associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. - name: cloud_services_cost description: Cloud service costs associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. + - name: cloud_services_credits + description: Cloud service credits associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. - name: query_cost description: Total cost associated with the query, calculated as sum of compute_cost and cloud_services_cost, in the primary currency of your account. + - name: query_credits + description: Total cost associated with the query, calculated as sum of compute_credits and cloud_services_credits, in the primary currency of your account. - name: ran_on_warehouse description: Indicator for whether the query ran on a warehouse. Certain queries, such as metadata queries, can be entirely processed in cloud services. - name: currency diff --git a/models/query_history_enriched.sql b/models/query_history_enriched.sql index 0fe47c1..6e3773a 100644 --- a/models/query_history_enriched.sql +++ b/models/query_history_enriched.sql @@ -43,8 +43,11 @@ cost_per_query as ( select cost_per_query.query_id, cost_per_query.compute_cost, + cost_per_query.compute_credits, cost_per_query.cloud_services_cost, + cost_per_query.cloud_services_credits, cost_per_query.query_cost, + cost_per_query.query_credits, cost_per_query.execution_start_time, -- Grab all columns from query_history (except the query time columns which we rename below) diff --git a/models/query_history_enriched.yml b/models/query_history_enriched.yml index 822dab7..8666bdf 100644 --- a/models/query_history_enriched.yml +++ b/models/query_history_enriched.yml @@ -156,10 +156,16 @@ models: description: When the query began executing on the warehouse (in the UTC time zone). This will always be after the start_time. - name: compute_cost description: Compute costs associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. + - name: compute_credits + description: Compute credits associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. - name: cloud_services_cost description: Cloud service costs associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. + - name: cloud_services_credits + description: Cloud service credits associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. - name: query_cost description: Total cost associated with the query, calculated as sum of compute_cost and cloud_services_cost, in the primary currency of your account. + - name: query_credits + description: Total credits associated with the query, calculated as sum of compute_credits and cloud_services_credits, in the primary currency of your account. - name: currency description: Spend currency, retrieved from Snowflake's daily rate sheet From 74b160c024a6741fff5c902aa3654cdaed0239f1 Mon Sep 17 00:00:00 2001 From: yingyingqiqi <48437786+yingyingqiqi@users.noreply.github.com> Date: Tue, 15 Aug 2023 01:36:49 +0000 Subject: [PATCH 2/3] update .changes --- .changes/unreleased/Features-20230815-013055.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230815-013055.yaml diff --git a/.changes/unreleased/Features-20230815-013055.yaml b/.changes/unreleased/Features-20230815-013055.yaml new file mode 100644 index 0000000..d319403 --- /dev/null +++ b/.changes/unreleased/Features-20230815-013055.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add credit consumption to cost per query +time: 2023-08-15T01:30:55.1004512Z +custom: + Author: yingyingqiqi + PR: "127" From 2e01b3e5c8bcad783c7dcda0b3443dc72fa718f1 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Thu, 24 Aug 2023 16:56:54 +0100 Subject: [PATCH 3/3] Apply suggestions from code review --- models/cost_per_query.yml | 6 +++--- models/query_history_enriched.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/cost_per_query.yml b/models/cost_per_query.yml index 62f18ba..80f07ee 100644 --- a/models/cost_per_query.yml +++ b/models/cost_per_query.yml @@ -19,15 +19,15 @@ models: - name: compute_cost description: Compute costs associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. - name: compute_credits - description: Compute credits associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. + description: Compute credits associated with the query. Can be 0 if the query did not run on a warehouse. - name: cloud_services_cost description: Cloud service costs associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. - name: cloud_services_credits - description: Cloud service credits associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. + description: Cloud service credits associated with the query. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. - name: query_cost description: Total cost associated with the query, calculated as sum of compute_cost and cloud_services_cost, in the primary currency of your account. - name: query_credits - description: Total cost associated with the query, calculated as sum of compute_credits and cloud_services_credits, in the primary currency of your account. + description: Total credits associated with the query, calculated as sum of compute_credits and cloud_services_credits. - name: ran_on_warehouse description: Indicator for whether the query ran on a warehouse. Certain queries, such as metadata queries, can be entirely processed in cloud services. - name: currency diff --git a/models/query_history_enriched.yml b/models/query_history_enriched.yml index 8666bdf..3483f23 100644 --- a/models/query_history_enriched.yml +++ b/models/query_history_enriched.yml @@ -157,15 +157,15 @@ models: - name: compute_cost description: Compute costs associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. - name: compute_credits - description: Compute credits associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. + description: Compute credits associated with the query. Can be 0 if the query did not run on a warehouse. - name: cloud_services_cost description: Cloud service costs associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. - name: cloud_services_credits - description: Cloud service credits associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. + description: Cloud service credits associated with the query. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. - name: query_cost description: Total cost associated with the query, calculated as sum of compute_cost and cloud_services_cost, in the primary currency of your account. - name: query_credits - description: Total credits associated with the query, calculated as sum of compute_credits and cloud_services_credits, in the primary currency of your account. + description: Total credits associated with the query, calculated as sum of compute_credits and cloud_services_credits. - name: currency description: Spend currency, retrieved from Snowflake's daily rate sheet