From 5915598ab77884f701669dacc9576d5a5cfd021e Mon Sep 17 00:00:00 2001 From: mattahrens Date: Thu, 2 Nov 2023 10:24:16 -0500 Subject: [PATCH 1/9] Updating dataproc container cost to be multiplied by number of cores Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py | 6 +++--- .../src/spark_rapids_pytools/cloud_api/dataproc_gke.py | 3 +-- user_tools/src/spark_rapids_pytools/cloud_api/onprem.py | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py index d3a24e648..45984e1bb 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py @@ -534,16 +534,16 @@ def _calculate_group_cost(self, cluster_inst: ClusterGetAccessor, node_type: Spa mem_gb = float(mem_mb) / 1024 cores_cost = self.price_provider.get_cpu_price(node_mc_type) * int(cores_count) memory_cost = self.price_provider.get_ram_price(node_mc_type) * mem_gb + dataproc_cost = self.price_provider.get_container_cost() * int(cores_count) # calculate the GPU cost gpu_per_machine, gpu_type = cluster_inst.get_gpu_per_node(node_type) gpu_cost = 0.0 if gpu_per_machine > 0: gpu_unit_price = self.price_provider.get_gpu_price(gpu_type) gpu_cost = gpu_unit_price * gpu_per_machine - return nodes_cnt * (cores_cost + memory_cost + gpu_cost) + return nodes_cnt * (cores_cost + memory_cost + dataproc_cost + gpu_cost) def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): master_cost = self._calculate_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self._calculate_group_cost(cluster, SparkNodeType.WORKER) - dataproc_cost = self.price_provider.get_container_cost() - return master_cost + workers_cost + dataproc_cost + return master_cost + workers_cost diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py index 06a71d342..65ad0754c 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py @@ -196,5 +196,4 @@ class DataprocGkeSavingsEstimator(DataprocSavingsEstimator): def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): master_cost = self._calculate_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self._calculate_group_cost(cluster, SparkNodeType.WORKER) - dataproc_gke_cost = self.price_provider.get_container_cost() - return master_cost + workers_cost + dataproc_gke_cost + return master_cost + workers_cost diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py index 28584617b..1e2b46e80 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py @@ -300,18 +300,18 @@ def __calculate_dataproc_group_cost(self, cluster_inst: ClusterGetAccessor, node cores_cost = self.price_provider.get_cpu_price(node_mc_type) * int(cores_count) memory_cost = self.price_provider.get_ram_price(node_mc_type) * mem_gb + dataproc_cost = self.price_provider.get_container_cost() * int(cores_count) # calculate the GPU cost gpu_per_machine, gpu_type = cluster_inst.get_gpu_per_node(node_type) gpu_cost = 0.0 if gpu_per_machine > 0: gpu_unit_price = self.price_provider.get_gpu_price(gpu_type) gpu_cost = gpu_unit_price * gpu_per_machine - return nodes_cnt * (cores_cost + memory_cost + gpu_cost) + return nodes_cnt * (cores_cost + memory_cost + dataproc_cost + gpu_cost) def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): if self.price_provider.name.casefold() == 'dataproc': master_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.WORKER) - dataproc_cost = self.price_provider.get_container_cost() - total_cost = master_cost + workers_cost + dataproc_cost + total_cost = master_cost + workers_cost return total_cost From 7dcd26ce299e06bfb0c903b9e0712702b1a2b2fb Mon Sep 17 00:00:00 2001 From: mattahrens Date: Thu, 2 Nov 2023 16:55:31 -0500 Subject: [PATCH 2/9] Simplifying changes for only Dataproc and not Dataproc GKE Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py | 8 +++++--- .../src/spark_rapids_pytools/cloud_api/dataproc_gke.py | 3 ++- user_tools/src/spark_rapids_pytools/cloud_api/onprem.py | 6 ++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py index 45984e1bb..15d466ab9 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc.py @@ -534,16 +534,18 @@ def _calculate_group_cost(self, cluster_inst: ClusterGetAccessor, node_type: Spa mem_gb = float(mem_mb) / 1024 cores_cost = self.price_provider.get_cpu_price(node_mc_type) * int(cores_count) memory_cost = self.price_provider.get_ram_price(node_mc_type) * mem_gb - dataproc_cost = self.price_provider.get_container_cost() * int(cores_count) # calculate the GPU cost gpu_per_machine, gpu_type = cluster_inst.get_gpu_per_node(node_type) gpu_cost = 0.0 if gpu_per_machine > 0: gpu_unit_price = self.price_provider.get_gpu_price(gpu_type) gpu_cost = gpu_unit_price * gpu_per_machine - return nodes_cnt * (cores_cost + memory_cost + dataproc_cost + gpu_cost) + return nodes_cnt * (cores_cost + memory_cost + gpu_cost) def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): master_cost = self._calculate_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self._calculate_group_cost(cluster, SparkNodeType.WORKER) - return master_cost + workers_cost + master_cores = cluster.get_nodes_cnt(SparkNodeType.MASTER) * cluster.get_node_core_count(SparkNodeType.MASTER) + worker_cores = cluster.get_nodes_cnt(SparkNodeType.WORKER) * cluster.get_node_core_count(SparkNodeType.WORKER) + dataproc_cost = self.price_provider.get_container_cost() * (master_cores + worker_cores) + return master_cost + workers_cost + dataproc_cost diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py index 65ad0754c..06a71d342 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py @@ -196,4 +196,5 @@ class DataprocGkeSavingsEstimator(DataprocSavingsEstimator): def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): master_cost = self._calculate_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self._calculate_group_cost(cluster, SparkNodeType.WORKER) - return master_cost + workers_cost + dataproc_gke_cost = self.price_provider.get_container_cost() + return master_cost + workers_cost + dataproc_gke_cost diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py index 1e2b46e80..cbd28ae5e 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py @@ -300,7 +300,6 @@ def __calculate_dataproc_group_cost(self, cluster_inst: ClusterGetAccessor, node cores_cost = self.price_provider.get_cpu_price(node_mc_type) * int(cores_count) memory_cost = self.price_provider.get_ram_price(node_mc_type) * mem_gb - dataproc_cost = self.price_provider.get_container_cost() * int(cores_count) # calculate the GPU cost gpu_per_machine, gpu_type = cluster_inst.get_gpu_per_node(node_type) gpu_cost = 0.0 @@ -313,5 +312,8 @@ def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): if self.price_provider.name.casefold() == 'dataproc': master_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.WORKER) - total_cost = master_cost + workers_cost + master_cores = cluster.get_nodes_cnt(SparkNodeType.MASTER) * cluster.get_node_core_count(SparkNodeType.MASTER) + worker_cores = cluster.get_nodes_cnt(SparkNodeType.WORKER) * cluster.get_node_core_count(SparkNodeType.WORKER) + dataproc_cost = self.price_provider.get_container_cost() * (master_cores + worker_cores) + total_cost = master_cost + workers_cost + dataproc_cost return total_cost From a9ceef6b38949f4ce23e5ae51e26402066e535a5 Mon Sep 17 00:00:00 2001 From: mattahrens Date: Fri, 3 Nov 2023 08:56:30 -0500 Subject: [PATCH 3/9] Fixing bug with extraneous dataproc_cost reference Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/onprem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py index cbd28ae5e..08cf4b231 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py @@ -306,7 +306,7 @@ def __calculate_dataproc_group_cost(self, cluster_inst: ClusterGetAccessor, node if gpu_per_machine > 0: gpu_unit_price = self.price_provider.get_gpu_price(gpu_type) gpu_cost = gpu_unit_price * gpu_per_machine - return nodes_cnt * (cores_cost + memory_cost + dataproc_cost + gpu_cost) + return nodes_cnt * (cores_cost + memory_cost + gpu_cost) def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): if self.price_provider.name.casefold() == 'dataproc': From 644a2a3aec2d9622acf0b27f6083a0e20273bd2b Mon Sep 17 00:00:00 2001 From: mattahrens Date: Fri, 3 Nov 2023 09:04:00 -0500 Subject: [PATCH 4/9] Fixing pylint with lines too long Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/onprem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py index 08cf4b231..2f7e9147e 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py @@ -312,8 +312,10 @@ def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): if self.price_provider.name.casefold() == 'dataproc': master_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.WORKER) - master_cores = cluster.get_nodes_cnt(SparkNodeType.MASTER) * cluster.get_node_core_count(SparkNodeType.MASTER) - worker_cores = cluster.get_nodes_cnt(SparkNodeType.WORKER) * cluster.get_node_core_count(SparkNodeType.WORKER) + master_cores = cluster.get_nodes_cnt(SparkNodeType.MASTER) * \ + cluster.get_node_core_count(SparkNodeType.MASTER) + worker_cores = cluster.get_nodes_cnt(SparkNodeType.WORKER) * \ + cluster.get_node_core_count(SparkNodeType.WORKER) dataproc_cost = self.price_provider.get_container_cost() * (master_cores + worker_cores) total_cost = master_cost + workers_cost + dataproc_cost return total_cost From 3697e16e1ae0cd5018327b03c6eb9529441e1e7c Mon Sep 17 00:00:00 2001 From: mattahrens Date: Fri, 3 Nov 2023 09:11:08 -0500 Subject: [PATCH 5/9] Fixing flake issue with indentation Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/onprem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py index 2f7e9147e..ca48e0a0f 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py @@ -313,9 +313,9 @@ def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): master_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.WORKER) master_cores = cluster.get_nodes_cnt(SparkNodeType.MASTER) * \ - cluster.get_node_core_count(SparkNodeType.MASTER) + cluster.get_node_core_count(SparkNodeType.MASTER) worker_cores = cluster.get_nodes_cnt(SparkNodeType.WORKER) * \ - cluster.get_node_core_count(SparkNodeType.WORKER) + cluster.get_node_core_count(SparkNodeType.WORKER) dataproc_cost = self.price_provider.get_container_cost() * (master_cores + worker_cores) total_cost = master_cost + workers_cost + dataproc_cost return total_cost From 34bb615d9e588d0175baaac53aa7ec40430bd6f3 Mon Sep 17 00:00:00 2001 From: mattahrens Date: Fri, 3 Nov 2023 09:28:53 -0500 Subject: [PATCH 6/9] Fixing flake issue with indentation Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/onprem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py index ca48e0a0f..a2993b712 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py @@ -312,10 +312,10 @@ def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): if self.price_provider.name.casefold() == 'dataproc': master_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.WORKER) - master_cores = cluster.get_nodes_cnt(SparkNodeType.MASTER) * \ - cluster.get_node_core_count(SparkNodeType.MASTER) - worker_cores = cluster.get_nodes_cnt(SparkNodeType.WORKER) * \ - cluster.get_node_core_count(SparkNodeType.WORKER) + master_cores = (cluster.get_nodes_cnt(SparkNodeType.MASTER) + * cluster.get_node_core_count(SparkNodeType.MASTER)) + worker_cores = (cluster.get_nodes_cnt(SparkNodeType.WORKER) + * cluster.get_node_core_count(SparkNodeType.WORKER)) dataproc_cost = self.price_provider.get_container_cost() * (master_cores + worker_cores) total_cost = master_cost + workers_cost + dataproc_cost return total_cost From 30d8381d290c87b18cd0a490dbc517f7bf1bba32 Mon Sep 17 00:00:00 2001 From: mattahrens Date: Fri, 3 Nov 2023 09:40:26 -0500 Subject: [PATCH 7/9] Fixing flake issue with whitespace Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/onprem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py index a2993b712..c2fc5f19b 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/onprem.py @@ -312,7 +312,7 @@ def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): if self.price_provider.name.casefold() == 'dataproc': master_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.MASTER) workers_cost = self.__calculate_dataproc_group_cost(cluster, SparkNodeType.WORKER) - master_cores = (cluster.get_nodes_cnt(SparkNodeType.MASTER) + master_cores = (cluster.get_nodes_cnt(SparkNodeType.MASTER) * cluster.get_node_core_count(SparkNodeType.MASTER)) worker_cores = (cluster.get_nodes_cnt(SparkNodeType.WORKER) * cluster.get_node_core_count(SparkNodeType.WORKER)) From a7592b4cf4252d4ea912ff5450b72e359eb04010 Mon Sep 17 00:00:00 2001 From: mattahrens Date: Fri, 3 Nov 2023 13:42:14 -0500 Subject: [PATCH 8/9] Fixing Dataproc GKE costs for dataproc container Signed-off-by: mattahrens --- .../src/spark_rapids_pytools/cloud_api/dataproc_gke.py | 5 ++--- .../src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py index 06a71d342..3243bd0e5 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py @@ -194,7 +194,6 @@ class DataprocGkeSavingsEstimator(DataprocSavingsEstimator): """ def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): - master_cost = self._calculate_group_cost(cluster, SparkNodeType.MASTER) - workers_cost = self._calculate_group_cost(cluster, SparkNodeType.WORKER) + dataproc_cost = super()._get_cost_per_cluster() dataproc_gke_cost = self.price_provider.get_container_cost() - return master_cost + workers_cost + dataproc_gke_cost + return dataproc_cost + dataproc_gke_cost diff --git a/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py b/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py index 5c01d3296..493da7e81 100644 --- a/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py +++ b/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py @@ -27,7 +27,6 @@ class DataprocGkePriceProvider(DataprocPriceProvider): name = 'DataprocGke' def get_container_cost(self) -> float: - dataproc_cost = super().get_container_cost() gke_container_cost = self.__get_gke_container_cost() return dataproc_cost + gke_container_cost From b4683c665b4527c0958952b118a01ec8eda83429 Mon Sep 17 00:00:00 2001 From: mattahrens Date: Fri, 3 Nov 2023 14:14:17 -0500 Subject: [PATCH 9/9] Fixing Dataproc GKE costs for dataproc container Signed-off-by: mattahrens --- user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py | 2 +- .../src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py index 3243bd0e5..6039914bd 100644 --- a/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py +++ b/user_tools/src/spark_rapids_pytools/cloud_api/dataproc_gke.py @@ -194,6 +194,6 @@ class DataprocGkeSavingsEstimator(DataprocSavingsEstimator): """ def _get_cost_per_cluster(self, cluster: ClusterGetAccessor): - dataproc_cost = super()._get_cost_per_cluster() + dataproc_cost = super()._get_cost_per_cluster(cluster) dataproc_gke_cost = self.price_provider.get_container_cost() return dataproc_cost + dataproc_gke_cost diff --git a/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py b/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py index 493da7e81..3c06e370b 100644 --- a/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py +++ b/user_tools/src/spark_rapids_pytools/pricing/dataproc_gke_pricing.py @@ -28,7 +28,7 @@ class DataprocGkePriceProvider(DataprocPriceProvider): def get_container_cost(self) -> float: gke_container_cost = self.__get_gke_container_cost() - return dataproc_cost + gke_container_cost + return gke_container_cost def __get_gke_container_cost(self) -> float: lookup_key = 'CP-GKE-CONTAINER-MANAGMENT-COST'