Skip to content

Commit

Permalink
fix yarn-client executor dispatch error; add debug log
Browse files Browse the repository at this point in the history
  • Loading branch information
kecookier committed Mar 23, 2024
1 parent 9cddfa7 commit 4c3a092
Showing 1 changed file with 6 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@ object UDFResolver extends Logging {
conf.put(BackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS, getAllLibraries(paths, sparkConf))
case None =>
}

logInfo(s"after resolve path " +
s"is:${sparkConf.getOption(BackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS)}")
}

// Try to unpack archive. Throws exception if failed.
Expand Down Expand Up @@ -177,7 +180,8 @@ object UDFResolver extends Logging {
val file = new File(f)
// Relative paths should be uploaded via --files or --archives
if (isRelativePath(f)) {
if (isYarnClient) {
logInfo(s"resolve relative path: $f")
if (isYarnClient && isDriver) {
throw new IllegalArgumentException(
"On yarn-client mode, driver only accepts absolute paths, but got " + f)
} else if (isYarnCluster) {
Expand All @@ -186,6 +190,7 @@ object UDFResolver extends Logging {
new File(SparkFiles.get(f))
}
} else {
logInfo(s"resolve absolute URI path: $f")
// Download or copy absolute paths to JniWorkspace.
val uri = Utils.resolveURI(f)
val name = file.getName
Expand Down

0 comments on commit 4c3a092

Please sign in to comment.