Merge branch 'dev' into spark-rapids-tools-484

# Conflicts:
#	user_tools/tests/spark_rapids_tools_ut/resources/cluster/databricks/test-azure-instances-catalog.json

.github/workflows/mvn-verify-check.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        spark-version: ['311', '320', '330', '341']
+        spark-version: ['311', '320', '333', '341']
     steps:
     - uses: actions/checkout@v3
 

core/README.md

@@ -20,14 +20,14 @@ mvn clean package
 ```
 
 After a successful build, the jar of 'rapids-4-spark-tools_2.12-*-SNAPSHOT.jar' will be in 'target/' directory.  
-This will build the plugin for a single version of Spark. By default, this is Apache Spark 3.1.1.
+This will build the plugin for a single version of Spark. By default, this is Apache Spark 3.3.3.
 
 For development purpose, you may need to run the tests against different spark versions.
 To run the tests against a specific Spark version, you can use the `-Dbuildver=XXX` command line option.  
-For instance to build Spark 3.3.0 you would use:
+For instance to build Spark 3.4.1 you would use:
 
 ```shell script
-mvn -Dbuildver=330 clean package
+mvn -Dbuildver=341 clean package
 ```
 
 Run `mvn help:all-profiles` to list supported Spark versions.

core/pom.xml

@@ -23,7 +23,7 @@
     <artifactId>rapids-4-spark-tools_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark tools</name>
     <description>RAPIDS Accelerator for Apache Spark tools</description>
-    <version>23.08.1-SNAPSHOT</version>
+    <version>23.08.2-SNAPSHOT</version>
     <packaging>jar</packaging>
     <url>http://github.com/NVIDIA/spark-rapids-tools</url>
 
@@ -73,7 +73,6 @@
         <profile>
             <id>release311</id>
             <activation>
-                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>buildver</name>
                     <value>311</value>
@@ -83,7 +82,7 @@
                 <buildver>311</buildver>
                 <spark.version>${spark311.version}</spark.version>
                 <delta.core.version>${delta10x.version}</delta.core.version>
-                <hadoop.version>3.2.0</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -98,7 +97,7 @@
                 <buildver>312</buildver>
                 <spark.version>${spark312.version}</spark.version>
                 <delta.core.version>${delta10x.version}</delta.core.version>
-                <hadoop.version>3.2.0</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -113,7 +112,22 @@
                 <buildver>313</buildver>
                 <spark.version>${spark313.version}</spark.version>
                 <delta.core.version>${delta10x.version}</delta.core.version>
-                <hadoop.version>3.2.0</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>release314</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>314</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>314</buildver>
+                <spark.version>${spark314.version}</spark.version>
+                <delta.core.version>${delta10x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -128,7 +142,7 @@
                 <buildver>320</buildver>
                 <spark.version>${spark320.version}</spark.version>
                 <delta.core.version>${delta20x.version}</delta.core.version>
-                <hadoop.version>3.3.1</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -143,7 +157,7 @@
                 <buildver>321</buildver>
                 <spark.version>${spark321.version}</spark.version>
                 <delta.core.version>${delta20x.version}</delta.core.version>
-                <hadoop.version>3.3.1</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -158,7 +172,7 @@
                 <buildver>322</buildver>
                 <spark.version>${spark322.version}</spark.version>
                 <delta.core.version>${delta20x.version}</delta.core.version>
-                <hadoop.version>3.3.1</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -173,7 +187,7 @@
                 <buildver>323</buildver>
                 <spark.version>${spark323.version}</spark.version>
                 <delta.core.version>${delta20x.version}</delta.core.version>
-                <hadoop.version>3.3.1</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -188,7 +202,7 @@
                 <buildver>324</buildver>
                 <spark.version>${spark324.version}</spark.version>
                 <delta.core.version>${delta20x.version}</delta.core.version>
-                <hadoop.version>3.3.1</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -203,7 +217,7 @@
                 <buildver>325</buildver>
                 <spark.version>${spark325.version}</spark.version>
                 <delta.core.version>${delta20x.version}</delta.core.version>
-                <hadoop.version>3.3.5</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -218,7 +232,7 @@
                 <buildver>330</buildver>
                 <spark.version>${spark330.version}</spark.version>
                 <delta.core.version>${delta23x.version}</delta.core.version>
-                <hadoop.version>3.3.2</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -233,7 +247,7 @@
                 <buildver>331</buildver>
                 <spark.version>${spark331.version}</spark.version>
                 <delta.core.version>${delta23x.version}</delta.core.version>
-                <hadoop.version>3.3.2</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -248,12 +262,13 @@
                 <buildver>332</buildver>
                 <spark.version>${spark332.version}</spark.version>
                 <delta.core.version>${delta23x.version}</delta.core.version>
-                <hadoop.version>3.3.2</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
             <id>release333</id>
             <activation>
+                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>buildver</name>
                     <value>333</value>
@@ -263,7 +278,22 @@
                 <buildver>333</buildver>
                 <spark.version>${spark333.version}</spark.version>
                 <delta.core.version>${delta23x.version}</delta.core.version>
-                <hadoop.version>3.3.4</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>release334</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>334</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>334</buildver>
+                <spark.version>${spark334.version}</spark.version>
+                <delta.core.version>${delta23x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -278,7 +308,7 @@
                 <buildver>340</buildver>
                 <spark.version>${spark340.version}</spark.version>
                 <delta.core.version>${delta24x.version}</delta.core.version>
-                <hadoop.version>3.3.4</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -293,7 +323,22 @@
                 <buildver>341</buildver>
                 <spark.version>${spark341.version}</spark.version>
                 <delta.core.version>${delta24x.version}</delta.core.version>
-                <hadoop.version>3.3.4</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>release342</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>342</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>342</buildver>
+                <spark.version>${spark342.version}</spark.version>
+                <delta.core.version>${delta24x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
         <profile>
@@ -308,14 +353,45 @@
                 <buildver>350</buildver>
                 <spark.version>${spark350.version}</spark.version>
                 <delta.core.version>${delta24x.version}</delta.core.version>
-                <hadoop.version>3.3.5</hadoop.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>release351</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>351</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>351</buildver>
+                <spark.version>${spark351.version}</spark.version>
+                <delta.core.version>${delta24x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>release400</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>400</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>400</buildver>
+                <spark.version>${spark400.version}</spark.version>
+                <delta.core.version>${delta24x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
     </profiles>
     <properties>
         <spark311.version>3.1.1</spark311.version>
         <spark312.version>3.1.2</spark312.version>
         <spark313.version>3.1.3</spark313.version>
+        <spark314.version>3.1.4-SNAPSHOT</spark314.version>
         <spark320.version>3.2.0</spark320.version>
         <spark321.version>3.2.1</spark321.version>
         <spark322.version>3.2.2</spark322.version>
@@ -325,16 +401,20 @@
         <spark330.version>3.3.0</spark330.version>
         <spark331.version>3.3.1</spark331.version>
         <spark332.version>3.3.2</spark332.version>
-        <spark333.version>3.3.3-SNAPSHOT</spark333.version>
+        <spark333.version>3.3.3</spark333.version>
+        <spark334.version>3.3.4-SNAPSHOT</spark334.version>
         <spark340.version>3.4.0</spark340.version>
         <spark341.version>3.4.1</spark341.version>
+        <spark342.version>3.4.2-SNAPSHOT</spark342.version>
         <spark350.version>3.5.0-SNAPSHOT</spark350.version>
+        <spark351.version>3.5.1-SNAPSHOT</spark351.version>
+        <spark400.version>4.0.0-SNAPSHOT</spark400.version>
         <scala.binary.version>2.12</scala.binary.version>
         <scala.plugin.version>4.3.0</scala.plugin.version>
         <scalatest-maven-plugin.version>2.0.2</scalatest-maven-plugin.version>
         <maven.clean.plugin.version>3.2.0</maven.clean.plugin.version>
         <scala.version>2.12.15</scala.version>
-        <spark.test.version>${spark311.version}</spark.test.version>
+        <spark.test.version>${spark333.version}</spark.test.version>
         <snakeyaml.version>2.0</snakeyaml.version>
         <scallop.version>3.5.1</scallop.version>
         <scalatest.version>3.0.5</scalatest.version>

core/src/main/resources/operatorsScore-databricks-aws.csv

@@ -155,6 +155,8 @@ MapKeys,2.45
 MapValues,2.45
 Max,2.45
 Md5,2.45
+MicrosToTimestamp,2.45
+MillisToTimestamp,2.45
 Min,2.45
 Minute,2.45
 MonotonicallyIncreasingID,2.45
@@ -193,6 +195,7 @@ RowNumber,2.45
 ScalaUDF,2.45
 ScalarSubquery,2.45
 Second,2.45
+SecondsToTimestamp,2.45
 Sequence,2.45
 ShiftLeft,2.45
 ShiftRight,2.45
@@ -217,6 +220,7 @@ StringRepeat,2.45
 StringReplace,2.45
 StringSplit,2.45
 StringToMap,2.45
+StringTranslate,2.45
 StringTrim,2.45
 StringTrimLeft,2.45
 StringTrimRight,2.45

core/src/main/resources/operatorsScore-databricks-azure.csv

@@ -155,6 +155,8 @@ MapKeys,2.73
 MapValues,2.73
 Max,2.73
 Md5,2.73
+MicrosToTimestamp,2.73
+MillisToTimestamp,2.73
 Min,2.73
 Minute,2.73
 MonotonicallyIncreasingID,2.73
@@ -193,6 +195,7 @@ RowNumber,2.73
 ScalaUDF,2.73
 ScalarSubquery,2.73
 Second,2.73
+SecondsToTimestamp,2.73
 Sequence,2.73
 ShiftLeft,2.73
 ShiftRight,2.73
@@ -217,6 +220,7 @@ StringRepeat,2.73
 StringReplace,2.73
 StringSplit,2.73
 StringToMap,2.73
+StringTranslate,2.73
 StringTrim,2.73
 StringTrimLeft,2.73
 StringTrimRight,2.73

core/src/main/resources/operatorsScore-dataproc-l4.csv

@@ -155,6 +155,8 @@ MapKeys,4.16
 MapValues,4.16
 Max,4.16
 Md5,4.16
+MicrosToTimestamp,4.16
+MillisToTimestamp,4.16
 Min,4.16
 Minute,4.16
 MonotonicallyIncreasingID,4.16
@@ -193,6 +195,7 @@ RowNumber,4.16
 ScalaUDF,4.16
 ScalarSubquery,4.16
 Second,4.16
+SecondsToTimestamp,4.16
 Sequence,4.16
 ShiftLeft,4.16
 ShiftRight,4.16
@@ -217,6 +220,7 @@ StringRepeat,4.16
 StringReplace,4.16
 StringSplit,4.16
 StringToMap,4.16
+StringTranslate,4.16
 StringTrim,4.16
 StringTrimLeft,4.16
 StringTrimRight,4.16

core/src/main/resources/operatorsScore-dataproc-t4.csv

@@ -155,6 +155,8 @@ MapKeys,4.88
 MapValues,4.88
 Max,4.88
 Md5,4.88
+MicrosToTimestamp,4.88
+MillisToTimestamp,4.88
 Min,4.88
 Minute,4.88
 MonotonicallyIncreasingID,4.88
@@ -193,6 +195,7 @@ RowNumber,4.88
 ScalaUDF,4.88
 ScalarSubquery,4.88
 Second,4.88
+SecondsToTimestamp,4.88
 Sequence,4.88
 ShiftLeft,4.88
 ShiftRight,4.88
@@ -217,6 +220,7 @@ StringRepeat,4.88
 StringReplace,4.88
 StringSplit,4.88
 StringToMap,4.88
+StringTranslate,4.88
 StringTrim,4.88
 StringTrimLeft,4.88
 StringTrimRight,4.88