From 68be7cb177fb5d51f7e80465805f41225ef5b131 Mon Sep 17 00:00:00 2001 From: Cindy Jiang <47068112+cindyyuanjiang@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:32:35 -0700 Subject: [PATCH] [FEA] Fix empty softwareProperties field in worker_info.yaml file for profiling tool (#623) * fixed null softwareProperties issue for clusterProperties Signed-off-by: cindyyuanjiang * refactored code Signed-off-by: cindyyuanjiang * added lod info for empty softwareProperties Signed-off-by: cindyyuanjiang --------- Signed-off-by: cindyyuanjiang --- .../com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala index 86909d98d..4241ac3c3 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala @@ -1035,7 +1035,12 @@ object AutoTuner extends Logging { representer.getPropertyUtils.setSkipMissingProperties(true) val constructor = new Constructor(classOf[ClusterProperties], new LoaderOptions()) val yamlObjNested = new Yaml(constructor, representer) - Option(yamlObjNested.load(clusterProps).asInstanceOf[ClusterProperties]) + val loadedClusterProps = yamlObjNested.load(clusterProps).asInstanceOf[ClusterProperties] + if (loadedClusterProps != null && loadedClusterProps.softwareProperties == null) { + logInfo("softwareProperties is empty from input worker_info file") + loadedClusterProps.softwareProperties = new util.LinkedHashMap[String, String]() + } + Option(loadedClusterProps) } def loadClusterProps(filePath: String): Option[ClusterProperties] = {