diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala index 02243ba37..1dc42b34a 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Analysis.scala @@ -390,7 +390,9 @@ class Analysis(apps: Seq[ApplicationInfo]) { val allRows = apps.flatMap { app => app.sqlIdToInfo.map { case (sqlId, sqlCase) => SQLDurationExecutorTimeProfileResult(app.index, app.appId, sqlId, sqlCase.duration, - sqlCase.hasDatasetOrRDD, app.appInfo.duration, sqlCase.problematic, + sqlCase.hasDatasetOrRDD, + if (app.appInfo != null) app.appInfo.duration else Option(0L), + sqlCase.problematic, sqlCase.sqlCpuTimePercent) } } diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala index 78a6cd5eb..126fa0bdd 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala @@ -34,13 +34,13 @@ case class StageMetrics(numTasks: Int, duration: String) class CollectInformation(apps: Seq[ApplicationInfo]) extends Logging { def getAppInfo: Seq[AppInfoProfileResults] = { - val allRows = apps.map { app => + val allRows = apps.filterNot(_.appInfo == null).map { app => val a = app.appInfo AppInfoProfileResults(app.index, a.appName, a.appId, a.sparkUser, a.startTime, a.endTime, a.duration, a.durationStr, a.sparkVersion, a.pluginEnabled) } - if (allRows.size > 0) { + if (allRows.nonEmpty) { allRows.sortBy(cols => (cols.appIndex)) } else { Seq.empty @@ -48,11 +48,11 @@ class CollectInformation(apps: Seq[ApplicationInfo]) extends Logging { } def getAppLogPath: Seq[AppLogPathProfileResults] = { - val allRows = apps.map { app => + val allRows = apps.filterNot(_.appInfo == null).map { app => val a = app.appInfo AppLogPathProfileResults(app.index, a.appName, a.appId, app.eventLogPath) } - if (allRows.size > 0) { + if (allRows.nonEmpty) { allRows.sortBy(cols => (cols.appIndex)) } else { Seq.empty