[SPARK-52181][SQL][FOLLOWUP] Avoid OOM in variant tests

### What changes were proposed in this pull request?

This is a follow-up of apache#50913 . It sets the variant size limit back to 16 MB in test environment, to reduce OOM in tests.

### Why are the changes needed?

make tests more reliable.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

existing tests

### Was this patch authored or co-authored using generative AI tooling?

no

Closes apache#50950 from cloud-fan/test.

Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: yangjie01 <[email protected]>

Author: cloud-fan

common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java

@@ -138,7 +138,9 @@ public class VariantUtil {
   public static final int U32_SIZE = 4;
 
   // Both variant value and variant metadata need to be no longer than 128MiB.
-  public static final int SIZE_LIMIT = 128 * 1024 * 1024;
+  // Note: to make tests more reliable, we set the max size to 16Mib to avoid OOM in tests.
+  public static final int SIZE_LIMIT =
+    System.getenv("SPARK_TESTING") != null ? U24_MAX + 1 : 128 * 1024 * 1024;
 
   public static final int MAX_DECIMAL4_PRECISION = 9;
   public static final int MAX_DECIMAL8_PRECISION = 18;

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtilsSuite.scala

@@ -133,9 +133,10 @@ class VariantExpressionEvalUtilsSuite extends SparkFunSuite {
       checkException(json, "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION",
         Map("badRecord" -> json, "failFastMode" -> "FAILFAST"))
     }
-    for (json <- Seq((0 to 32 * 1024 * 1024).mkString("[", ",", "]"))) {
+    for (json <- Seq("\"" + "a" * (16 * 1024 * 1024) + "\"",
+      (0 to 4 * 1024 * 1024).mkString("[", ",", "]"))) {
       checkException(json, "VARIANT_SIZE_LIMIT",
-        Map("sizeLimit" -> "128.0 MiB", "functionName" -> "`parse_json`"))
+        Map("sizeLimit" -> "16.0 MiB", "functionName" -> "`parse_json`"))
     }
   }
 

sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala

@@ -824,8 +824,8 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
       parameters = Map("schema" -> "\"STRUCT<a: VARIANT, b: VARIANT>\""))
 
     // In singleVariantColumn mode, from_csv normally treats all inputs as valid. The only exception
-    // case is the input exceeds the variant size limit (128MiB).
-    val largeInput = "a" * (128 * 1024 * 1024)
+    // case is the input exceeds the variant size limit (16MiB).
+    val largeInput = "a" * (16 * 1024 * 1024)
     checkAnswer(
       Seq(largeInput).toDF("value").select(
         from_csv(

sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala

@@ -135,8 +135,7 @@ class VariantEndToEndSuite extends QueryTest with SharedSparkSession {
     check("{1:2}", null)
     check("{\"a\":1", null)
     check("{\"a\":[a,b,c]}", null)
-    check("\"" + "a" * (16 * 1024 * 1024) + "\"")
-    check("\"" + "a" * (128 * 1024 * 1024) + "\"", null)
+    check("\"" + "a" * (16 * 1024 * 1024) + "\"", null)
   }
 
   test("to_json with nested variant") {