diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StringPromotionTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StringPromotionTypeCoercion.scala index 5ca3736d2352b..d6e3df4442454 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StringPromotionTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/StringPromotionTypeCoercion.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.types.{ DoubleType, NullType, StringTypeExpression, + TimestampLTZNanosTypeExpression, TimestampType, TimestampTypeExpression } @@ -55,10 +56,20 @@ object StringPromotionTypeCoercion { // For equality between string and timestamp we cast the string to a timestamp // so that things like rounding of subsecond precision does not affect the comparison. + // Both micros and nanos are scoped to the LTZ family (TimestampType / TimestampLTZNanosType): + // the NTZ families need no arm here, because their equality reaches the general + // BinaryComparison arm below and findCommonTypeForBinaryComparison returns the config-blind + // nanos/micros common type for them -- the same Cast this arm would add. Only the LTZ arm is + // load-bearing, since without it LTZ equality would take the range path and be promoted to + // string under legacy castDatetimeToString. case p @ Equality(left @ StringTypeExpression(), right @ TimestampTypeExpression()) => p.withNewChildren(Seq(Cast(left, TimestampType), right)) case p @ Equality(left @ TimestampTypeExpression(), right @ StringTypeExpression()) => p.withNewChildren(Seq(left, Cast(right, TimestampType))) + case p @ Equality(left @ StringTypeExpression(), right @ TimestampLTZNanosTypeExpression()) => + p.withNewChildren(Seq(Cast(left, right.dataType), right)) + case p @ Equality(left @ TimestampLTZNanosTypeExpression(), right @ StringTypeExpression()) => + p.withNewChildren(Seq(left, Cast(right, left.dataType))) case p @ BinaryComparison(left, right) if findCommonTypeForBinaryComparison(left.dataType, right.dataType, conf).isDefined => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 53de166e69edf..7c76e692bb5ba 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -145,6 +145,14 @@ object TypeCoercion extends TypeCoercionBase { => if (conf.castDatetimeToString) Some(st) else Some(TimestampType) case (TimestampType, st: StringType) => if (conf.castDatetimeToString) Some(st) else Some(TimestampType) + // Mirror the micros TimestampType (LTZ) arms above: nanos TIMESTAMP_LTZ(p) honors + // castDatetimeToString. Nanos TIMESTAMP_NTZ(p) intentionally has no arm here and, exactly like + // micros TimestampNTZType, falls through to the config-blind canPromoteAsInBinaryComparison + // line below -- so the LTZ and NTZ families stay consistent across micros and nanos. + case (st: StringType, tsNanos: TimestampLTZNanosType) + => if (conf.castDatetimeToString) Some(st) else Some(tsNanos) + case (tsNanos: TimestampLTZNanosType, st: StringType) + => if (conf.castDatetimeToString) Some(st) else Some(tsNanos) case (st: StringType, NullType) => Some(st) case (NullType, st: StringType) => Some(st) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala index 6cb6b7d429d85..329af882391d3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala @@ -87,6 +87,11 @@ private[sql] object AnyTimestampNanoTypeExpression { e.dataType.isInstanceOf[AnyTimestampNanoType] } +private[sql] object TimestampLTZNanosTypeExpression { + def unapply(e: Expression): Boolean = + e.dataType.isInstanceOf[TimestampLTZNanosType] +} + private[sql] object DecimalExpression { def unapply(e: Expression): Option[(Int, Int)] = e.dataType match { case t: DecimalType => Some((t.precision, t.scale)) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala index d990a393ff653..2fbea06b4d066 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala @@ -1701,6 +1701,66 @@ class TypeCoercionSuite extends TypeCoercionSuiteBase { EqualTo(Cast(date0301, TimestampType), timestamp0301000000)) ruleTest(rule, LessThan(date0301, timestamp0301000001), LessThan(Cast(date0301, TimestampType), timestamp0301000001)) + + // SPARK-57811: a string operand is coerced to the nanosecond timestamp type in comparisons + // and predicates, mirroring the microsecond timestamp handling above. The concrete operand + // type (family + precision) is preserved, so the string is cast to that exact nanos type. + Seq(7, 8, 9).foreach { p => + Seq(TimestampLTZNanosType(p), TimestampNTZNanosType(p)).foreach { nt => + val tsn = AttributeReference("tsn", nt)() + val strLit = Literal("2020-01-02 03:04:05.123456789") + // Equality path: the string is cast to the nanos operand's own type so subsecond rounding + // does not affect the comparison. LTZ takes the explicit StringPromotionTypeCoercion + // Equality arm; NTZ has no arm and reaches the same cast via the general BinaryComparison + // fall-through (findCommonTypeForBinaryComparison returns the config-blind nanos type), + // mirroring how micros TimestampType (arm) vs TimestampNTZType (fall-through) are handled. + // The `Equality` extractor matches both 3VL `EqualTo` and null-safe `EqualNullSafe`, so + // both are covered. + ruleTest(rule, EqualTo(tsn, strLit), EqualTo(tsn, Cast(strLit, nt))) + ruleTest(rule, EqualTo(strLit, tsn), EqualTo(Cast(strLit, nt), tsn)) + ruleTest(rule, EqualNullSafe(tsn, strLit), EqualNullSafe(tsn, Cast(strLit, nt))) + ruleTest(rule, EqualNullSafe(strLit, tsn), EqualNullSafe(Cast(strLit, nt), tsn)) + // Range path (findCommonTypeForBinaryComparison). + ruleTest(rule, LessThan(tsn, strLit), LessThan(tsn, Cast(strLit, nt))) + ruleTest(rule, GreaterThanOrEqual(strLit, tsn), + GreaterThanOrEqual(Cast(strLit, nt), tsn)) + } + } + + // SPARK-57811: under legacy `castDatetimeToString`, the two nanos families mirror their micros + // counterparts. LTZ has a range arm in findCommonTypeForBinaryComparison, so its range + // comparisons promote both operands to string (like micros TimestampType); NTZ has no arm and + // stays config-blind (like micros TimestampNTZType), casting the string to the nanos type. In + // both families equality still casts the string to nanos, because the Equality arm fires before + // the range arm and reads no config. This block is the assertion that distinguishes the new + // production arms: in the default config the generic string-promotion fall-through already + // yields the nanos common type, so only the legacy branch separates the LTZ arm's effect. + withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") { + val ltzType = TimestampLTZNanosType(9) + val ltz = AttributeReference("ltz", ltzType)() + val ntzType = TimestampNTZNanosType(9) + val ntz = AttributeReference("ntz", ntzType)() + val strLit = Literal("2020-01-02 03:04:05.123456789") + // LTZ range: both operands become strings (matches micros TimestampType). + ruleTest(rule, LessThan(ltz, strLit), LessThan(Cast(ltz, StringType), strLit)) + ruleTest(rule, GreaterThanOrEqual(strLit, ltz), + GreaterThanOrEqual(strLit, Cast(ltz, StringType))) + // NTZ range: config-blind, the string is cast to the nanos type (matches micros + // TimestampNTZType, which has no arm and falls through to canPromoteAsInBinaryComparison). + ruleTest(rule, LessThan(ntz, strLit), LessThan(ntz, Cast(strLit, ntzType))) + ruleTest(rule, GreaterThanOrEqual(strLit, ntz), + GreaterThanOrEqual(Cast(strLit, ntzType), ntz)) + // Equality: for both families the string is still cast to nanos so subseconds are compared + // exactly -- LTZ via the explicit Equality arm (which fires before the range arm), NTZ via + // the config-blind BinaryComparison fall-through. Holds for both 3VL `EqualTo` and null-safe + // `EqualNullSafe`. + Seq(ltzType -> ltz, ntzType -> ntz).foreach { case (nt, tsn) => + ruleTest(rule, EqualTo(tsn, strLit), EqualTo(tsn, Cast(strLit, nt))) + ruleTest(rule, EqualTo(strLit, tsn), EqualTo(Cast(strLit, nt), tsn)) + ruleTest(rule, EqualNullSafe(tsn, strLit), EqualNullSafe(tsn, Cast(strLit, nt))) + ruleTest(rule, EqualNullSafe(strLit, tsn), EqualNullSafe(Cast(strLit, nt), tsn)) + } + } } test("cast WindowFrame boundaries to the type they operate upon") { diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out index 454eeddf2f23c..a39c1e9f5ac3f 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out @@ -1036,6 +1036,63 @@ Sort [v#x ASC NULLS FIRST], true +- OneRowRelation +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c = '2020-01-02 03:04:05.123456788', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query analysis +Project [(c#x = cast(2020-01-02 03:04:05.123456789 as timestamp_ltz(9))) AS (c = 2020-01-02 03:04:05.123456789)#x, (c#x = cast(2020-01-02 03:04:05.123456788 as timestamp_ltz(9))) AS (c = 2020-01-02 03:04:05.123456788)#x, (c#x < cast(2020-01-02 03:04:05.123456790 as timestamp_ltz(9))) AS (c < 2020-01-02 03:04:05.123456790)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + +-- !query +SELECT c FROM VALUES + (TIMESTAMP_LTZ '2020-01-02 03:04:05.000000001'), + (TIMESTAMP_LTZ '2020-01-02 03:04:05.000000009') AS t(c) + WHERE c BETWEEN '2020-01-02 03:04:05.000000001' AND '2020-01-02 03:04:05.000000005' +-- !query analysis +Project [c#x] ++- Filter between(c#x, 2020-01-02 03:04:05.000000001, 2020-01-02 03:04:05.000000005) + +- SubqueryAlias t + +- LocalRelation [c#x] + + +-- !query +SET spark.sql.ansi.enabled=false +-- !query analysis +SetCommand (spark.sql.ansi.enabled,Some(false)) + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=true +-- !query analysis +SetCommand (spark.sql.legacy.typeCoercion.datetimeToString.enabled,Some(true)) + + +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query analysis +Project [(c#x = cast(2020-01-02 03:04:05.123456789 as timestamp_ltz(9))) AS (c = 2020-01-02 03:04:05.123456789)#x, (cast(c#x as string) < 2020-01-02 03:04:05.123456790) AS (c < 2020-01-02 03:04:05.123456790)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=false +-- !query analysis +SetCommand (spark.sql.legacy.typeCoercion.datetimeToString.enabled,Some(false)) + + +-- !query +SET spark.sql.ansi.enabled=true +-- !query analysis +SetCommand (spark.sql.ansi.enabled,Some(true)) + + -- !query SELECT date_trunc('SECOND', TIMESTAMP_LTZ '2020-01-01 12:34:56.123456789 UTC') -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out index 879c084a74968..c3cdf2c7738f9 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out @@ -870,6 +870,63 @@ Sort [v#x ASC NULLS FIRST], true +- OneRowRelation +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c = '2020-01-02 03:04:05.123456788', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query analysis +Project [(c#x = cast(2020-01-02 03:04:05.123456789 as timestamp_ntz(9))) AS (c = 2020-01-02 03:04:05.123456789)#x, (c#x = cast(2020-01-02 03:04:05.123456788 as timestamp_ntz(9))) AS (c = 2020-01-02 03:04:05.123456788)#x, (c#x < cast(2020-01-02 03:04:05.123456790 as timestamp_ntz(9))) AS (c < 2020-01-02 03:04:05.123456790)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + +-- !query +SELECT c FROM VALUES + (TIMESTAMP_NTZ '2020-01-02 03:04:05.000000001'), + (TIMESTAMP_NTZ '2020-01-02 03:04:05.000000009') AS t(c) + WHERE c BETWEEN '2020-01-02 03:04:05.000000001' AND '2020-01-02 03:04:05.000000005' +-- !query analysis +Project [c#x] ++- Filter between(c#x, 2020-01-02 03:04:05.000000001, 2020-01-02 03:04:05.000000005) + +- SubqueryAlias t + +- LocalRelation [c#x] + + +-- !query +SET spark.sql.ansi.enabled=false +-- !query analysis +SetCommand (spark.sql.ansi.enabled,Some(false)) + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=true +-- !query analysis +SetCommand (spark.sql.legacy.typeCoercion.datetimeToString.enabled,Some(true)) + + +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query analysis +Project [(c#x = cast(2020-01-02 03:04:05.123456789 as timestamp_ntz(9))) AS (c = 2020-01-02 03:04:05.123456789)#x, (c#x < cast(2020-01-02 03:04:05.123456790 as timestamp_ntz(9))) AS (c < 2020-01-02 03:04:05.123456790)#x] ++- SubqueryAlias t + +- LocalRelation [c#x] + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=false +-- !query analysis +SetCommand (spark.sql.legacy.typeCoercion.datetimeToString.enabled,Some(false)) + + +-- !query +SET spark.sql.ansi.enabled=true +-- !query analysis +SetCommand (spark.sql.ansi.enabled,Some(true)) + + -- !query SELECT date_trunc('SECOND', TIMESTAMP_NTZ '2020-01-01 12:34:56.123456789') -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql index a6bed71cc1ac0..849252ad09968 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql @@ -329,6 +329,36 @@ SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM ( UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100' UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY v; +-- SPARK-57811: a string operand is coerced to the nanosecond timestamp type in comparisons and +-- predicates (not truncated to micros, not promoted to string). The 9th fractional digit is +-- significant, so an off-by-one-nanosecond literal does not compare equal. +SELECT c = '2020-01-02 03:04:05.123456789', + c = '2020-01-02 03:04:05.123456788', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789') AS t(c); + +-- BETWEEN over nanosecond timestamps: only the value inside the sub-microsecond range qualifies. +SELECT c FROM VALUES + (TIMESTAMP_LTZ '2020-01-02 03:04:05.000000001'), + (TIMESTAMP_LTZ '2020-01-02 03:04:05.000000009') AS t(c) + WHERE c BETWEEN '2020-01-02 03:04:05.000000001' AND '2020-01-02 03:04:05.000000005'; + +-- SPARK-57811: this is the config that exercises the new non-ANSI production arms. The arms live +-- in TypeCoercion (not AnsiTypeCoercion), so ANSI must be disabled; and only under legacy +-- castDatetimeToString does the range path differ. With both flags set, the range comparison +-- promotes BOTH operands to string (Cast(c AS STRING), matching the micro TIMESTAMP type), while +-- equality still casts the string to the nanos type (the Equality arm fires before the range arm). +-- In every other config (ANSI on, or ANSI off without the legacy flag) the string coerces to the +-- nanos type, identical to the parent commit, which is why the default-config cases above do not +-- distinguish this change. +SET spark.sql.ansi.enabled=false; +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=true; +SELECT c = '2020-01-02 03:04:05.123456789', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789') AS t(c); +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=false; +SET spark.sql.ansi.enabled=true; + -- SPARK-57821: date_trunc keeps the nanosecond type/family and zeroes the whole fraction (including -- the sub-microsecond digits); MICROSECOND keeps epochMicros and only drops nanosWithinMicro. LTZ -- truncates in the session time zone (America/Los_Angeles, UTC-08:00), so DAY over an early-hours diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql index aa173fb726204..4639360771d68 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql @@ -269,6 +269,34 @@ SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM ( UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100' UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY v; +-- SPARK-57811: a string operand is coerced to the nanosecond timestamp type in comparisons and +-- predicates (not truncated to micros, not promoted to string). The 9th fractional digit is +-- significant, so an off-by-one-nanosecond literal does not compare equal. +SELECT c = '2020-01-02 03:04:05.123456789', + c = '2020-01-02 03:04:05.123456788', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789') AS t(c); + +-- BETWEEN over nanosecond timestamps: only the value inside the sub-microsecond range qualifies. +SELECT c FROM VALUES + (TIMESTAMP_NTZ '2020-01-02 03:04:05.000000001'), + (TIMESTAMP_NTZ '2020-01-02 03:04:05.000000009') AS t(c) + WHERE c BETWEEN '2020-01-02 03:04:05.000000001' AND '2020-01-02 03:04:05.000000005'; + +-- SPARK-57811: TIMESTAMP_NTZ(p) mirrors micros TimestampNTZType under legacy castDatetimeToString. +-- Micros TimestampNTZType has no arm in findCommonTypeForBinaryComparison, so it stays config-blind +-- and casts the string to the timestamp type even under the legacy flag; nanos NTZ has no arm +-- either and does the same. (Only the LTZ family, like micros TimestampType, promotes the range +-- comparison to string under this flag -- see timestamp-ltz-nanos.sql.) So with both flags set the +-- NTZ range comparison still casts the string to the nanos type, identical to the default config. +SET spark.sql.ansi.enabled=false; +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=true; +SELECT c = '2020-01-02 03:04:05.123456789', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789') AS t(c); +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=false; +SET spark.sql.ansi.enabled=true; + -- SPARK-57821: date_trunc keeps the nanosecond type/family and zeroes the whole fraction (including -- the sub-microsecond digits); MICROSECOND keeps epochMicros and only drops nanosWithinMicro. NTZ -- is zone-independent, so DAY/HOUR read the wall clock and never shift. diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out index 8aa3586481e77..74274f55951e5 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out @@ -1127,6 +1127,70 @@ struct 2020-01-01 00:00:00.0000009 NULL +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c = '2020-01-02 03:04:05.123456788', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query schema +struct<(c = 2020-01-02 03:04:05.123456789):boolean,(c = 2020-01-02 03:04:05.123456788):boolean,(c < 2020-01-02 03:04:05.123456790):boolean> +-- !query output +true false true + + +-- !query +SELECT c FROM VALUES + (TIMESTAMP_LTZ '2020-01-02 03:04:05.000000001'), + (TIMESTAMP_LTZ '2020-01-02 03:04:05.000000009') AS t(c) + WHERE c BETWEEN '2020-01-02 03:04:05.000000001' AND '2020-01-02 03:04:05.000000005' +-- !query schema +struct +-- !query output +2020-01-02 03:04:05.000000001 + + +-- !query +SET spark.sql.ansi.enabled=false +-- !query schema +struct +-- !query output +spark.sql.ansi.enabled false + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=true +-- !query schema +struct +-- !query output +spark.sql.legacy.typeCoercion.datetimeToString.enabled true + + +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query schema +struct<(c = 2020-01-02 03:04:05.123456789):boolean,(c < 2020-01-02 03:04:05.123456790):boolean> +-- !query output +true true + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=false +-- !query schema +struct +-- !query output +spark.sql.legacy.typeCoercion.datetimeToString.enabled false + + +-- !query +SET spark.sql.ansi.enabled=true +-- !query schema +struct +-- !query output +spark.sql.ansi.enabled true + + -- !query SELECT date_trunc('SECOND', TIMESTAMP_LTZ '2020-01-01 12:34:56.123456789 UTC') -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out index acb2fe1951155..3f6c2dd9f384b 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out @@ -924,6 +924,70 @@ struct 2020-01-01 00:00:00.0000009 NULL +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c = '2020-01-02 03:04:05.123456788', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query schema +struct<(c = 2020-01-02 03:04:05.123456789):boolean,(c = 2020-01-02 03:04:05.123456788):boolean,(c < 2020-01-02 03:04:05.123456790):boolean> +-- !query output +true false true + + +-- !query +SELECT c FROM VALUES + (TIMESTAMP_NTZ '2020-01-02 03:04:05.000000001'), + (TIMESTAMP_NTZ '2020-01-02 03:04:05.000000009') AS t(c) + WHERE c BETWEEN '2020-01-02 03:04:05.000000001' AND '2020-01-02 03:04:05.000000005' +-- !query schema +struct +-- !query output +2020-01-02 03:04:05.000000001 + + +-- !query +SET spark.sql.ansi.enabled=false +-- !query schema +struct +-- !query output +spark.sql.ansi.enabled false + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=true +-- !query schema +struct +-- !query output +spark.sql.legacy.typeCoercion.datetimeToString.enabled true + + +-- !query +SELECT c = '2020-01-02 03:04:05.123456789', + c < '2020-01-02 03:04:05.123456790' + FROM VALUES (TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789') AS t(c) +-- !query schema +struct<(c = 2020-01-02 03:04:05.123456789):boolean,(c < 2020-01-02 03:04:05.123456790):boolean> +-- !query output +true true + + +-- !query +SET spark.sql.legacy.typeCoercion.datetimeToString.enabled=false +-- !query schema +struct +-- !query output +spark.sql.legacy.typeCoercion.datetimeToString.enabled false + + +-- !query +SET spark.sql.ansi.enabled=true +-- !query schema +struct +-- !query output +spark.sql.ansi.enabled true + + -- !query SELECT date_trunc('SECOND', TIMESTAMP_NTZ '2020-01-01 12:34:56.123456789') -- !query schema diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosWideningSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosWideningSuiteBase.scala index 22bb69fdfc6b4..523b980adceec 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosWideningSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosWideningSuiteBase.scala @@ -132,6 +132,29 @@ abstract class TimestampNanosWideningSuiteBase extends SharedSparkSession { val ntzEq = twoCols(TimestampNTZType, ldtA, TimestampNTZNanosType(9), ldtA) checkAnswer(ntzEq.selectExpr("a = b", "a < b"), Row(true, false)) } + + test("SPARK-57811: string operand coerces to the nanosecond timestamp type") { + // Equality on the exact sub-microsecond value: the string is parsed at nanos precision, so a + // string that differs only in the 9th digit does NOT compare equal. The paired + // `= '...789' => true` / `= '...788' => false` is the load-bearing pair: it rules out + // micro-truncation (truncating the operand to micros would make both strings equal the column). + // The nanos-vs-string-promotion distinction is locked separately by the analyzer goldens + // (explicit `cast as timestamp_ntz(9)`) and the unit tests in TypeCoercionSuite. + val ntz = + single(TimestampNTZNanosType(9), LocalDateTime.parse("2020-01-02T03:04:05.123456789")) + checkAnswer( + ntz.selectExpr( + "c = '2020-01-02 03:04:05.123456789'", + "c = '2020-01-02 03:04:05.123456788'", + "c < '2020-01-02 03:04:05.123456790'", + "c > '2020-01-02 03:04:05.123456788'"), + Row(true, false, true, true)) + + // LTZ operand: the string literal is parsed in the session zone (America/Los_Angeles), so the + // 2020-01-01T00:00:00Z instant equals the local wall-clock time eight hours earlier. + val ltz = single(TimestampLTZNanosType(9), Instant.parse("2020-01-01T00:00:00Z")) + checkAnswer(ltz.selectExpr("c = '2019-12-31 16:00:00'"), Row(true)) + } } // Runs the nanosecond timestamp widening tests with ANSI mode enabled explicitly.