Merge dev into main

Signed-off-by: spark-rapids automation <[email protected]>

.github/workflows/configuration.json

@@ -6,7 +6,7 @@
     },
     {
       "title": "### Core",
-      "labels": ["tools"]
+      "labels": ["core_tools"]
     },
     {
       "title": "### Miscellaneous",

.pre-commit-config.yaml

@@ -31,8 +31,8 @@ repos:
     rev: v4.0.1
     hooks:
       - id: check-added-large-files
-        name: Check for file over 2.0MiB
-        args: ['--maxkb=2000', '--enforce-all']
+        name: Check for file over 4.0MiB
+        args: ['--maxkb=4000', '--enforce-all']
       - id: trailing-whitespace
         name: trim trailing white spaces preserving md files
         args: ['--markdown-linebreak-ext=md']

.pylintrc

@@ -56,6 +56,8 @@ disable=
         # R0913: Too many arguments
         too-many-arguments,
         # R0914: Too many local variables
+        too-many-positional-arguments,
+        # R0917: Too many positional arguments
         too-many-locals,
         # R0801: Similar lines in 2 files
         duplicate-code,

core/README.md

@@ -32,6 +32,17 @@ mvn -Dbuildver=351 clean package
 
 Run `mvn help:all-profiles` to list supported Spark versions.
 
+### Running tests
+
+The unit tests are run by default when building unless they are explicitly skipped by specifying `-DskipTests`.
+
+To run an individual test the `-Dsuites` option can be specified:
+
+```bash
+mvn test -Dsuites=com.nvidia.spark.rapids.tool.qualification.QualificationSuite
+```
+
+
 ### Setting up an Integrated Development Environment
 
 Before proceeding with importing spark-rapids-tools into IDEA or switching to a different Spark release

core/pom.xml

@@ -23,7 +23,7 @@
     <artifactId>rapids-4-spark-tools_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark tools</name>
     <description>RAPIDS Accelerator for Apache Spark tools</description>
-    <version>24.08.2</version>
+    <version>24.08.3-SNAPSHOT</version>
     <packaging>jar</packaging>
     <url>http://github.com/NVIDIA/spark-rapids-tools</url>
 

core/src/main/resources/operatorsScore-databricks-aws-a10G.csv

@@ -304,3 +304,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-databricks-aws-t4.csv

@@ -304,3 +304,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-databricks-azure-t4.csv

@@ -292,3 +292,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-dataproc-gke-l4.csv

@@ -286,3 +286,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-dataproc-gke-t4.csv

@@ -286,3 +286,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-dataproc-l4.csv

@@ -292,3 +292,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv

@@ -286,3 +286,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-dataproc-t4.csv

@@ -292,3 +292,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-emr-a10.csv

@@ -292,3 +292,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-emr-a10G.csv

@@ -292,3 +292,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-emr-t4.csv

@@ -292,3 +292,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/operatorsScore-onprem-a100.csv

@@ -304,3 +304,4 @@ MapFromArrays,1.5
 DecimalSum,1.5
 MaxBy,1.5
 MinBy,1.5
+ArrayJoin,1.5

core/src/main/resources/photonOperatorMappings/databricks-13_3.json

@@ -0,0 +1,106 @@
+{
+  "//comments": [
+    "This file contains the mapping between Photon operators and Spark operators generated from Databricks Runtime 13.3",
+    "Some entries have one-to-many mappings. For example, 'PhotonAgg' can map to either 'HashAggregate', 'SortAggregate', or 'ObjectHashAggregate'.",
+    "Currently, only the first mapping in the list is used.",
+    "This limitation exists because we cannot differentiate between these operators in the SparkPlan.",
+    "TODO: Create separate mapping file for different Photon/Databricks versions"
+  ],
+  "PhotonAdapter": [
+    "Scan"
+  ],
+  "PhotonScan": [
+    "Scan"
+  ],
+  "PhotonSubqueryBroadcast": [
+    "SubqueryBroadcast"
+  ],
+  "PhotonResultStage": [
+    "WholeStageCodegen"
+  ],
+  "PhotonUnionShuffleExchangeSink": [
+    "Union"
+  ],
+  "PhotonUnionShuffleMapStage": [
+    "WholeStageCodegen"
+  ],
+  "PhotonTopK": [
+    "TakeOrderedAndProject"
+  ],
+  "PhotonAgg": [
+    "HashAggregate",
+    "SortAggregate",
+    "ObjectHashAggregate"
+  ],
+  "PhotonBroadcastExchange": [
+    "BroadcastExchange"
+  ],
+  "PhotonBroadcastHashJoin": [
+    "BroadcastHashJoin"
+  ],
+  "PhotonBroadcastNestedLoopJoin": [
+    "BroadcastNestedLoopJoin"
+  ],
+  "PhotonExpand": [
+    "Expand"
+  ],
+  "PhotonFilter": [
+    "Filter"
+  ],
+  "PhotonGenerate": [
+    "Generate"
+  ],
+  "PhotonGlobalLimit": [
+    "GlobalLimit"
+  ],
+  "PhotonGroupingAgg": [
+    "HashAggregate",
+    "SortAggregate",
+    "ObjectHashAggregate"
+  ],
+  "PhotonGroupingAggWithRollup": [
+    "HashAggregate"
+  ],
+  "PhotonHashJoin": [
+    "BroadcastHashJoin"
+  ],
+  "PhotonLocalLimit": [
+    "LocalLimit"
+  ],
+  "PhotonProject": [
+    "Project"
+  ],
+  "PhotonRowToColumnar": [
+    "RowToColumnar"
+  ],
+  "PhotonShuffledHashJoin": [
+    "SortMergeJoin"
+  ],
+  "PhotonShuffleExchangeSink": [
+    "Exchange",
+    "StageBoundary",
+    "BroadcastExchange"
+  ],
+  "PhotonShuffleExchangeSource": [
+    "Exchange",
+    "StageBoundary",
+    "AQEShuffleRead",
+    "ShuffleQueryStage"
+  ],
+  "PhotonShuffleHashJoin": [
+    "ShuffledHashJoin"
+  ],
+  "PhotonShuffleMapStage": [
+    "WholeStageCodegen"
+  ],
+  "PhotonSort": [
+    "Sort"
+  ],
+  "PhotonUnion": [
+    "Union"
+  ],
+  "PhotonWindow": [
+    "Window",
+    "RunningWindowFunction"
+  ]
+}

core/src/main/resources/supportedExprs.csv

@@ -42,6 +42,10 @@ ArrayFilter,S,`filter`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,N
 ArrayIntersect,S,`array_intersect`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA
 ArrayIntersect,S,`array_intersect`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA
 ArrayIntersect,S,`array_intersect`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA
+ArrayJoin,S,`array_join`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA
+ArrayJoin,S,`array_join`,None,project,delimiter,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
+ArrayJoin,S,`array_join`,None,project,nullReplacement,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
+ArrayJoin,S,`array_join`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
 ArrayMax,S,`array_max`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA
 ArrayMax,S,`array_max`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA
 ArrayMin,S,`array_min`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA

core/src/main/scala/com/nvidia/spark/rapids/SparkRapidsBuildInfoEvent.scala

@@ -17,6 +17,7 @@
 package com.nvidia.spark.rapids
 
 import org.apache.spark.scheduler.SparkListenerEvent
+import org.apache.spark.sql.rapids.tool.annotation.ToolsReflection
 
 
 /**
@@ -30,4 +31,9 @@ case class SparkRapidsBuildInfoEvent(
   sparkRapidsJniBuildInfo: Map[String, String],
   cudfBuildInfo: Map[String, String],
   sparkRapidsPrivateBuildInfo: Map[String, String]
-) extends SparkListenerEvent
+) extends SparkListenerEvent {
+  @ToolsReflection("BD-3.2.1", "Ignore")
+  override val eventTime: Long = 0
+  @ToolsReflection("BD-3.2.1", "Ignore")
+  override val eventType: String = ""
+}