Updated CI and Nightly pipelines with Spark 3.5, fixed incompatible t…

…ests

azure-pipelines-e2e-tests-template.yml

@@ -122,14 +122,14 @@ stages:
             script: |
               echo "Download Hadoop utils for Windows."
               $hadoopBinaryUrl = "https://github.com/steveloughran/winutils/releases/download/tag_2017-08-29-hadoop-2.8.1-native/hadoop-2.8.1.zip"
-              # Spark 3.3.3 version binary use Hadoop3 dependency
-              if ("3.3.3" -contains "${{ test.version }}") {
+              # Spark 3.3.0+ version binary uses Hadoop3 dependency
+              if ([version]"3.3.0" -le [version]"${{ test.version }}") {
                 $hadoopBinaryUrl = "https://github.com/SparkSnail/winutils/releases/download/hadoop-3.3.5/hadoop-3.3.5.zip"
               }
               curl -k -L -o hadoop.zip $hadoopBinaryUrl
               Expand-Archive -Path hadoop.zip -Destination .
               New-Item -ItemType Directory -Force -Path hadoop\bin
-              if ("3.3.3" -contains "${{ test.version }}") {
+              if ([version]"3.3.0" -le [version]"${{ test.version }}") {
                 cp hadoop-3.3.5\winutils.exe hadoop\bin
                 # Hadoop 3.3 need to add hadoop.dll to environment varibles to avoid UnsatisfiedLinkError
                 cp hadoop-3.3.5\hadoop.dll hadoop\bin
@@ -142,12 +142,8 @@ stages:
         - pwsh: |
             echo "Downloading Spark ${{ test.version }}"
             $sparkBinaryName = "spark-${{ test.version }}-bin-hadoop2.7"
-            # In spark 3.3.0, 3.3.1, 3.3.2, 3.3.4, the binary name with hadoop2 dependency has changed to spark-${{ test.version }}-bin-hadoop2.tgz
-            if ("3.3.0", "3.3.1", "3.3.2", "3.3.4" -contains "${{ test.version }}") {
-                $sparkBinaryName = "spark-${{ test.version }}-bin-hadoop2"
-            }
-            # In spark 3.3.3, the binary don't provide hadoop2 version, so we use hadoop3 version
-            if ("3.3.3" -contains "${{ test.version }}") {
+            # Spark 3.3.0+ uses Hadoop3
+            if ([version]"3.3.0" -le [version]"${{ test.version }}") {
                 $sparkBinaryName = "spark-${{ test.version }}-bin-hadoop3"
             }
             curl -k -L -o spark-${{ test.version }}.tgz https://archive.apache.org/dist/spark/spark-${{ test.version }}/${sparkBinaryName}.tgz

azure-pipelines-pr.yml

@@ -31,7 +31,7 @@ variables:
   backwardCompatibleTestOptions_Linux_3_1: ""
   forwardCompatibleTestOptions_Linux_3_1: ""
 
-  # Skip all forward/backward compatibility tests since Spark 3.2 is not supported before this release.
+  # Skip all forward/backward compatibility tests since Spark 3.2 and 3.5 are not supported before this release.
   backwardCompatibleTestOptions_Windows_3_2: "--filter FullyQualifiedName=NONE"
   forwardCompatibleTestOptions_Windows_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
   backwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
@@ -41,6 +41,11 @@ variables:
   forwardCompatibleTestOptions_Windows_3_3: $(backwardCompatibleTestOptions_Windows_3_3)
   backwardCompatibleTestOptions_Linux_3_3: $(backwardCompatibleTestOptions_Windows_3_3)
   forwardCompatibleTestOptions_Linux_3_3: $(backwardCompatibleTestOptions_Windows_3_3)
+
+  backwardCompatibleTestOptions_Windows_3_5: "--filter FullyQualifiedName=NONE"
+  forwardCompatibleTestOptions_Windows_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  backwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  forwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
 
   # Azure DevOps variables are transformed into environment variables, with these variables we
   # avoid the first time experience and telemetry to speed up the build.
@@ -73,6 +78,11 @@ parameters:
   - '3.3.2'
   - '3.3.3'
   - '3.3.4'
+  - '3.5.0'
+  - '3.5.1'
+  - '3.5.2'
+  - '3.5.3'
+
 # List of OS types to run E2E tests, run each test in both 'Windows' and 'Linux' environments
 - name: listOfE2ETestsPoolTypes
   type: object

azure-pipelines.yml

@@ -31,12 +31,17 @@ variables:
   backwardCompatibleTestOptions_Linux_3_1: ""
   forwardCompatibleTestOptions_Linux_3_1: ""
 
-  # Skip all forward/backward compatibility tests since Spark 3.2 is not supported before this release.
+  # Skip all forward/backward compatibility tests since Spark 3.2 and 3.5 are not supported before this release.
   backwardCompatibleTestOptions_Windows_3_2: "--filter FullyQualifiedName=NONE"
   forwardCompatibleTestOptions_Windows_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
   backwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
   forwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
 
+  backwardCompatibleTestOptions_Windows_3_5: "--filter FullyQualifiedName=NONE"
+  forwardCompatibleTestOptions_Windows_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  backwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  forwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+
   # Azure DevOps variables are transformed into environment variables, with these variables we
   # avoid the first time experience and telemetry to speed up the build.
   DOTNET_CLI_TELEMETRY_OPTOUT: 1
@@ -413,3 +418,51 @@ stages:
         testOptions: ""
         backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
         forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
+    - version: '3.5.0'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)
+    - version: '3.5.1'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)
+    - version: '3.5.2'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)
+    - version: '3.5.3'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)

src/csharp/Microsoft.Spark.E2ETest/IpcTests/SparkContextTests.cs

@@ -57,16 +57,22 @@ public void TestSignaturesV2_4_X()
 
         /// <summary>
         /// Test signatures for APIs introduced in Spark 3.1.*.
+        /// In Spark 3.5 Spark throws an exception when trying to delete
+        /// archive.zip from temp folder, and causes failures of other tests
         /// </summary>
-        [SkipIfSparkVersionIsLessThan(Versions.V3_1_0)]
+        [SkipIfSparkVersionIsNotInRange(Versions.V3_1_0, Versions.V3_3_0)]
         public void TestSignaturesV3_1_X()
         {
             SparkContext sc = SparkContext.GetOrCreate(new SparkConf());
 
             string archivePath = $"{TestEnvironment.ResourceDirectory}archive.zip";
+
             sc.AddArchive(archivePath);
 
-            Assert.IsType<string[]>(sc.ListArchives().ToArray());
+            var archives = sc.ListArchives().ToArray();
+
+            Assert.IsType<string[]>(archives);
+            Assert.NotEmpty(archives.Where(a => a.EndsWith("archive.zip")));
         }
     }
 }

src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/CatalogTests.cs

@@ -59,7 +59,6 @@ public void TestSignaturesV2_4_X()
                 Assert.IsType<bool>(catalog.FunctionExists("functionname"));
                 Assert.IsType<Database>(catalog.GetDatabase("default"));
                 Assert.IsType<Function>(catalog.GetFunction("abs"));
-                Assert.IsType<Function>(catalog.GetFunction(null, "abs"));
                 Assert.IsType<Table>(catalog.GetTable("users"));
                 Assert.IsType<Table>(catalog.GetTable("default", "users"));
                 Assert.IsType<bool>(catalog.IsCached("users"));

src/csharp/Microsoft.Spark.Worker.UnitTest/PayloadWriter.cs

@@ -351,6 +351,7 @@ internal PayloadWriter Create(Version version = null)
                         new BroadcastVariableWriterV2_4_X(),
                         new CommandWriterV2_4_X());
                 case Versions.V3_3_0:
+                case Versions.V3_5_1:
                     return new PayloadWriter(
                         version,
                         new TaskContextWriterV3_3_X(),