From 3582fb7af241d235231166406f1a6fce1e502465 Mon Sep 17 00:00:00 2001 From: cindyyuanjiang Date: Wed, 18 Oct 2023 18:27:13 -0700 Subject: [PATCH 1/3] fixed null softwareProperties issue for clusterProperties Signed-off-by: cindyyuanjiang --- .../com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala index 86909d98d..53a7090e4 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala @@ -1035,7 +1035,11 @@ object AutoTuner extends Logging { representer.getPropertyUtils.setSkipMissingProperties(true) val constructor = new Constructor(classOf[ClusterProperties], new LoaderOptions()) val yamlObjNested = new Yaml(constructor, representer) - Option(yamlObjNested.load(clusterProps).asInstanceOf[ClusterProperties]) + val clusterPropsOpt = Option(yamlObjNested.load(clusterProps).asInstanceOf[ClusterProperties]) + if (clusterPropsOpt.isDefined && clusterPropsOpt.get.softwareProperties == null) { + clusterPropsOpt.get.softwareProperties = new util.LinkedHashMap[String, String]() + } + clusterPropsOpt } def loadClusterProps(filePath: String): Option[ClusterProperties] = { From 5d60350ebd637b35a77b67406a9634cb12fa7d3a Mon Sep 17 00:00:00 2001 From: cindyyuanjiang Date: Wed, 18 Oct 2023 18:44:27 -0700 Subject: [PATCH 2/3] refactored code Signed-off-by: cindyyuanjiang --- .../nvidia/spark/rapids/tool/profiling/AutoTuner.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala index 53a7090e4..42fd35bd1 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala @@ -1035,11 +1035,11 @@ object AutoTuner extends Logging { representer.getPropertyUtils.setSkipMissingProperties(true) val constructor = new Constructor(classOf[ClusterProperties], new LoaderOptions()) val yamlObjNested = new Yaml(constructor, representer) - val clusterPropsOpt = Option(yamlObjNested.load(clusterProps).asInstanceOf[ClusterProperties]) - if (clusterPropsOpt.isDefined && clusterPropsOpt.get.softwareProperties == null) { - clusterPropsOpt.get.softwareProperties = new util.LinkedHashMap[String, String]() + val loadedClusterProps = yamlObjNested.load(clusterProps).asInstanceOf[ClusterProperties] + if (loadedClusterProps != null && loadedClusterProps.softwareProperties == null) { + loadedClusterProps.softwareProperties = new util.LinkedHashMap[String, String]() } - clusterPropsOpt + Option(loadedClusterProps) } def loadClusterProps(filePath: String): Option[ClusterProperties] = { From 57dfc6bb53411b6811446863a03a0b3afa69daf4 Mon Sep 17 00:00:00 2001 From: cindyyuanjiang Date: Fri, 20 Oct 2023 17:10:08 -0700 Subject: [PATCH 3/3] added lod info for empty softwareProperties Signed-off-by: cindyyuanjiang --- .../scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala index 42fd35bd1..4241ac3c3 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala @@ -1037,6 +1037,7 @@ object AutoTuner extends Logging { val yamlObjNested = new Yaml(constructor, representer) val loadedClusterProps = yamlObjNested.load(clusterProps).asInstanceOf[ClusterProperties] if (loadedClusterProps != null && loadedClusterProps.softwareProperties == null) { + logInfo("softwareProperties is empty from input worker_info file") loadedClusterProps.softwareProperties = new util.LinkedHashMap[String, String]() } Option(loadedClusterProps)