From 363620a5ed1ddbdc70365965d3dab1e9a09f5b0e Mon Sep 17 00:00:00 2001 From: Stevo Mitric Date: Wed, 12 Aug 2026 12:45:22 +0000 Subject: [PATCH 1/2] [SPARK-56822][SQL][TESTS] Add nanosecond-timestamp coverage for collect_set `collect_set` deduplicates via a `HashSet` keyed on the physical `TimestampNanosVal`, whose `equals`/`hashCode` cover the full `(epochMicros, nanosWithinMicro)` pair, so sub-microsecond-distinct values are kept distinct; the result element type is exactly `child.dataType`. It already works over the nanosecond timestamp types (`TIMESTAMP_NTZ(p)` / `TIMESTAMP_LTZ(p)`, p in [7, 9]) but had no dedicated test coverage. Add an end-to-end test to `TimestampNanosFunctionsSuiteBase` (run in both ANSI modes) asserting that collect_set deduplicates on the full nanosecond value: sub-microsecond-distinct values stay distinct at p=9 but collapse when floored below the grid at p in [7, 8]. Also add a deterministic golden SQL case (`sort_array(collect_set(...))`) to `timestamp-ntz-nanos.sql` / `timestamp-ltz-nanos.sql`. Test-only; no production change. Co-authored-by: Isaac --- .../timestamp-ltz-nanos.sql.out | 11 +++++++ .../timestamp-ntz-nanos.sql.out | 11 +++++++ .../sql-tests/inputs/timestamp-ltz-nanos.sql | 10 ++++++ .../sql-tests/inputs/timestamp-ntz-nanos.sql | 9 +++++ .../results/timestamp-ltz-nanos.sql.out | 11 +++++++ .../results/timestamp-ntz-nanos.sql.out | 11 +++++++ .../TimestampNanosFunctionsSuiteBase.scala | 33 +++++++++++++++++++ 7 files changed, 96 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out index cca0a1e7d836d..ca4838e31569f 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out @@ -740,6 +740,17 @@ Sort [c#x ASC NULLS FIRST], true +- LocalRelation [c#x] +-- !query +SELECT sort_array(collect_set(c)) FROM VALUES + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c) +-- !query analysis +Aggregate [sort_array(collect_set(c#x, 0, 0, true), true) AS sort_array(collect_set(c), true)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + -- !query SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789') -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out index d8ff98fbe7606..2944a24b050f5 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out @@ -668,6 +668,17 @@ Sort [c#x ASC NULLS FIRST], true +- LocalRelation [c#x] +-- !query +SELECT sort_array(collect_set(c)) FROM VALUES + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c) +-- !query analysis +Aggregate [sort_array(collect_set(c#x, 0, 0, true), true) AS sort_array(collect_set(c), true)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + -- !query SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789') -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql index 9133dda2e0064..d6d7efc1b12cd 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql @@ -220,6 +220,16 @@ SELECT c, count(*) FROM VALUES (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c) GROUP BY c ORDER BY c; +-- SPARK-56822: collect_set over nanosecond-precision TIMESTAMP_LTZ. It deduplicates on the full +-- sub-microsecond value: the two .000000001 rows collapse to one, the .000000999 row stays, so the +-- sorted set has two distinct elements and the element type stays TIMESTAMP_LTZ(9); values render +-- in the session time zone (America/Los_Angeles). collect_set order is non-deterministic, so the +-- output is stabilized with sort_array. +SELECT sort_array(collect_set(c)) FROM VALUES + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c); + -- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision values. The result is -- whole-second BIGINT; the sub-second digits are dropped. A literal without an explicit zone is -- read in the session time zone (America/Los_Angeles, UTC-08:00); an explicit-zone literal fixes diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql index d145ecb7146a5..4fbc56d88ac84 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql @@ -193,6 +193,15 @@ SELECT c, count(*) FROM VALUES (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c) GROUP BY c ORDER BY c; +-- SPARK-56822: collect_set over nanosecond-precision TIMESTAMP_NTZ. It deduplicates on the full +-- sub-microsecond value: the two .000000001 rows collapse to one, the .000000999 row stays, so the +-- sorted set has two distinct elements and the element type stays TIMESTAMP_NTZ(9). collect_set +-- order is non-deterministic, so the output is stabilized with sort_array. +SELECT sort_array(collect_set(c)) FROM VALUES + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c); + -- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision values. The result is -- whole-second BIGINT; the sub-second digits are dropped and NTZ applies no zone shift, so the -- wall-clock value is read as the epoch instant. diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out index 31a40e0d80a02..b5a67236d5740 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out @@ -832,6 +832,17 @@ struct 2019-12-31 16:00:00.000000999 1 +-- !query +SELECT sort_array(collect_set(c)) FROM VALUES + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'), + (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c) +-- !query schema +struct> +-- !query output +[2019-12-31 16:00:00.000000001,2019-12-31 16:00:00.000000999] + + -- !query SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789') -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out index a39b70a3d4deb..161b804855d0f 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out @@ -750,6 +750,17 @@ struct 2020-01-01 00:00:00.000000999 1 +-- !query +SELECT sort_array(collect_set(c)) FROM VALUES + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'), + (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c) +-- !query schema +struct> +-- !query output +[2020-01-01 00:00:00.000000001,2020-01-01 00:00:00.000000999] + + -- !query SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789') -- !query schema diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala index 3f15cb2ef3cd6..038cdae53d5cb 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala @@ -710,6 +710,39 @@ abstract class TimestampNanosFunctionsSuiteBase extends SharedSparkSession { } } + // collect_set over nanosecond-precision timestamps (SPARK-56822). `CollectSet` deduplicates via a + // `HashSet` keyed on the physical `TimestampNanosVal`, whose `equals`/`hashCode` cover the full + // `(epochMicros, nanosWithinMicro)` pair, so sub-microsecond-distinct values are kept distinct; + // the result element type is exactly `child.dataType`. + + test("SPARK-56822: collect_set over nanos deduplicates on the full sub-microsecond value") { + // Two values share the microsecond and differ only in the last nanosecond digit. Flooring the + // input to precision `p` collapses them when p < 9 (the distinguishing digit is below the grid), + // but keeps them apart at p = 9. collect_set must dedup on the full stored value, not on micros. + Seq(7, 8, 9).foreach { p => + val schema = new StructType().add("ntz", TimestampNTZNanosType(p)) + val data = Seq( + Row(LocalDateTime.parse("2020-01-01T12:34:56.123456780")), + Row(LocalDateTime.parse("2020-01-01T12:34:56.123456789")), + Row(LocalDateTime.parse("2020-01-01T12:34:56.123456780"))) + val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema) + + val res = df.selectExpr("collect_set(ntz)") + assert(res.schema.head.dataType === ArrayType(TimestampNTZNanosType(p), containsNull = false)) + + // Values are floored to `p` on ingestion, so the distinct set depends on `p`. + val expected = p match { + case 7 => Set(LocalDateTime.parse("2020-01-01T12:34:56.123456700")) + case 8 => Set(LocalDateTime.parse("2020-01-01T12:34:56.123456780")) + case _ => Set( + LocalDateTime.parse("2020-01-01T12:34:56.123456780"), + LocalDateTime.parse("2020-01-01T12:34:56.123456789")) + } + val collected = res.collect().head.getSeq[LocalDateTime](0).toSet + assert(collected === expected, s"collect_set(p=$p) expected $expected, got $collected") + } + } + test("SPARK-57816: date_format / to_char / to_varchar over nanosecond-precision timestamps") { // The 9-`S` pattern is a fixed-width fraction field, so it always emits 9 digits; truncating to // precision `p` zeros the low digits (floor); it does not drop them. The session zone is From db4c605a321436c94eb68b8e3beb0b8dc6fc06f4 Mon Sep 17 00:00:00 2001 From: Stevo Mitric Date: Thu, 13 Aug 2026 09:37:06 +0000 Subject: [PATCH 2/2] [SPARK-56822][SQL][TESTS] Fix Scalastyle line-length violations Reflow over-long comment lines in TimestampNanosFunctionsSuiteBase to stay within the 100-character limit. Comment-only change. Co-authored-by: Isaac --- .../apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala index 038cdae53d5cb..cee6340b3f68f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala @@ -717,8 +717,9 @@ abstract class TimestampNanosFunctionsSuiteBase extends SharedSparkSession { test("SPARK-56822: collect_set over nanos deduplicates on the full sub-microsecond value") { // Two values share the microsecond and differ only in the last nanosecond digit. Flooring the - // input to precision `p` collapses them when p < 9 (the distinguishing digit is below the grid), - // but keeps them apart at p = 9. collect_set must dedup on the full stored value, not on micros. + // input to precision `p` collapses them when p < 9 (the distinguishing digit is below the + // grid), but keeps them apart at p = 9. collect_set must dedup on the full stored value, + // not on micros. Seq(7, 8, 9).foreach { p => val schema = new StructType().add("ntz", TimestampNTZNanosType(p)) val data = Seq(