-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Unified platform handling and fetching of operator score files (#661)
* Handle incorrect platform in core tools Signed-off-by: Partho Sarthi <[email protected]> * Add platform factory Signed-off-by: Partho Sarthi <[email protected]> * Add comments for default GPU types. Signed-off-by: Partho Sarthi <[email protected]> * Improve platform names in documentation Signed-off-by: Partho Sarthi <[email protected]> * Add docs Signed-off-by: Partho Sarthi <[email protected]> * Replace Map with pattern matching and rename PlatformTypes to PlatformNames Signed-off-by: Partho Sarthi <[email protected]> * Rename argument to platformName Signed-off-by: Partho Sarthi <[email protected]> --------- Signed-off-by: Partho Sarthi <[email protected]>
- Loading branch information
Showing
9 changed files
with
170 additions
and
124 deletions.
There are no files selected for viewing
File renamed without changes.
156 changes: 156 additions & 0 deletions
156
core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| /* | ||
| * Copyright (c) 2023, NVIDIA CORPORATION. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.nvidia.spark.rapids.tool | ||
|
|
||
| import org.apache.spark.internal.Logging | ||
|
|
||
| /** | ||
| * Utility object containing constants for various platform names. | ||
| */ | ||
| object PlatformNames { | ||
| val DATABRICKS_AWS = "databricks-aws" | ||
| val DATABRICKS_AZURE = "databricks-azure" | ||
| val DATAPROC = "dataproc" | ||
| val DATAPROC_GKE_L4 = "dataproc-gke-l4" | ||
| val DATAPROC_GKE_T4 = "dataproc-gke-t4" | ||
| val DATAPROC_L4 = "dataproc-l4" | ||
| val DATAPROC_SL_L4 = "dataproc-serverless-l4" | ||
| val DATAPROC_T4 = "dataproc-t4" | ||
| val EMR = "emr" | ||
| val EMR_A10 = "emr-a10" | ||
| val EMR_T4 = "emr-t4" | ||
| val ONPREM = "onprem" | ||
|
|
||
| /** | ||
| * Return a list of all platform names. | ||
| */ | ||
| def getAllNames: List[String] = List( | ||
| DATABRICKS_AWS, DATABRICKS_AZURE, DATAPROC, DATAPROC_GKE_L4, DATAPROC_GKE_T4, | ||
| DATAPROC_L4, DATAPROC_SL_L4, DATAPROC_T4, EMR, EMR_A10, EMR_T4, ONPREM | ||
| ) | ||
| } | ||
|
|
||
| /** | ||
| * Represents a platform and its associated recommendations. | ||
| * | ||
| * @param platformName Name of the platform. See [[PlatformNames]] for supported platform names. | ||
| */ | ||
| class Platform(platformName: String) { | ||
| /** | ||
| * Recommendations to be excluded from the list of recommendations. | ||
| * These have the highest priority. | ||
| */ | ||
| val recommendationsToExclude: Seq[String] = Seq.empty | ||
| /** | ||
| * Recommendations to be included in the final list of recommendations. | ||
| * These properties should be specific to the platform and not general Spark properties. | ||
| * For example: "spark.databricks.optimizer.dynamicFilePruning" for the Databricks platform. | ||
| * | ||
| * Represented as a tuple of (propertyKey, propertyValue). | ||
| */ | ||
| val recommendationsToInclude: Seq[(String, String)] = Seq.empty | ||
| /** | ||
| * Dynamically calculates the recommendation for a specific Spark property by invoking | ||
| * the appropriate function based on `sparkProperty`. | ||
| * TODO: Implement this function and integrate with existing code in AutoTuner | ||
| * | ||
| * @param sparkProperty The Spark property for which the recommendation is calculated. | ||
| * @param args Variable list of arguments passed to the calculation function for dynamic | ||
| * processing. | ||
| * @return Optional string containing the recommendation, or `None` if unavailable. | ||
| */ | ||
| def getRecommendation(sparkProperty: String, args: Any*): Option[String] = None | ||
|
|
||
| /** | ||
| * Checks if the `property` is valid: | ||
| * 1. It should not be in exclusion list | ||
| * OR | ||
| * 2. It should be in the inclusion list | ||
| */ | ||
| def isValidRecommendation(property: String): Boolean = { | ||
| !recommendationsToExclude.contains(property) || | ||
| recommendationsToInclude.map(_._1).contains(property) | ||
| } | ||
|
|
||
| /** | ||
| * Checks if the `comment` is valid: | ||
| * 1. It should not have any property from the exclusion list | ||
| */ | ||
| def isValidComment(comment: String): Boolean = { | ||
| recommendationsToExclude.forall(excluded => !comment.contains(excluded)) | ||
| } | ||
|
|
||
| def getOperatorScoreFile: String = { | ||
| s"operatorsScore-$platformName.csv" | ||
| } | ||
| } | ||
|
|
||
| class DatabricksPlatform(platformType: String) extends Platform(platformType) { | ||
| override val recommendationsToExclude: Seq[String] = Seq( | ||
| "spark.executor.cores", | ||
| "spark.executor.instances", | ||
| "spark.executor.memory", | ||
| "spark.executor.memoryOverhead" | ||
| ) | ||
| override val recommendationsToInclude: Seq[(String, String)] = Seq( | ||
| ("spark.databricks.optimizer.dynamicFilePruning", "false") | ||
| ) | ||
| } | ||
|
|
||
| class DataprocPlatform(platformType: String) extends Platform(platformType) | ||
|
|
||
| class EmrPlatform(platformType: String) extends Platform(platformType) | ||
|
|
||
| class OnPremPlatform extends Platform(PlatformNames.ONPREM) | ||
|
|
||
| /** | ||
| * Factory for creating instances of different platforms. | ||
| * This factory supports various platforms and provides methods for creating | ||
| * corresponding platform instances. | ||
| */ | ||
| object PlatformFactory extends Logging { | ||
| /** | ||
| * Creates an instance of a platform based on the specified platform key. | ||
| * If platform key is not defined, returns an instance of onprem platform. | ||
| * | ||
| * @param platformKey The key representing the desired platform. | ||
| * @return An instance of the specified platform. | ||
| * @throws IllegalArgumentException if the specified platform key is not supported. | ||
| */ | ||
| def createInstance(platformKey: String): Platform = { | ||
| platformKey match { | ||
| case PlatformNames.DATABRICKS_AWS => new DatabricksPlatform(PlatformNames.DATABRICKS_AWS) | ||
| case PlatformNames.DATABRICKS_AZURE => new DatabricksPlatform(PlatformNames.DATABRICKS_AZURE) | ||
| case PlatformNames.DATAPROC | PlatformNames.DATAPROC_T4 => | ||
| // if no GPU specified, then default to dataproc-t4 for backward compatibility | ||
| new DataprocPlatform(PlatformNames.DATAPROC_T4) | ||
| case PlatformNames.DATAPROC_L4 => new DataprocPlatform(PlatformNames.DATAPROC_L4) | ||
| case PlatformNames.DATAPROC_SL_L4 => new DataprocPlatform(PlatformNames.DATAPROC_SL_L4) | ||
| case PlatformNames.DATAPROC_GKE_L4 => new DataprocPlatform(PlatformNames.DATAPROC_GKE_L4) | ||
| case PlatformNames.DATAPROC_GKE_T4 => new DataprocPlatform(PlatformNames.DATAPROC_GKE_T4) | ||
| case PlatformNames.EMR | PlatformNames.EMR_T4 => | ||
| // if no GPU specified, then default to emr-t4 for backward compatibility | ||
| new EmrPlatform(PlatformNames.EMR_T4) | ||
| case PlatformNames.EMR_A10 => new EmrPlatform(PlatformNames.EMR_A10) | ||
| case PlatformNames.ONPREM => new OnPremPlatform | ||
| case p if p.isEmpty => | ||
| logInfo(s"Platform is not specified. Using ${PlatformNames.ONPREM} as default.") | ||
| new OnPremPlatform | ||
| case _ => throw new IllegalArgumentException(s"Unsupported platform: $platformKey. " + | ||
| s"Options include ${PlatformNames.getAllNames.mkString(", ")}.") | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
83 changes: 0 additions & 83 deletions
83
core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Platform.scala
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters