diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index 6df35deef8075..baaf876583c36 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -7787,6 +7787,11 @@
         "message" : [
           "The following session configuration(s) are incompatible with Real-Time Mode: <invalidReasons>. Update or remove them and restart the query."
         ]
+      },
+      "STATEFUL_OPERATORS_BEFORE_UNION_NOT_SUPPORTED" : {
+        "message" : [
+          "Streaming queries in real-time mode cannot include stateful operators (e.g. aggregate, deduplicate, transformWithState) before a union. Please restructure your query to apply the union before any stateful operations."
+        ]
       }
     },
     "sqlState" : "0A000"
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRealTimeModeSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRealTimeModeSuite.scala
index ae23b53fc35ad..4524bdb8faa25 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRealTimeModeSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRealTimeModeSuite.scala
@@ -18,23 +18,50 @@
 package org.apache.spark.sql.kafka010
 
 import java.util.UUID
+import java.util.regex.Pattern
 
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.{SparkConf, SparkContext, SparkIllegalStateException}
-import org.apache.spark.sql.execution.datasources.v2.LowLatencyClock
+import org.apache.spark.sql.Encoders
+import org.apache.spark.sql.execution.datasources.v2.{LowLatencyClock, RealTimeStreamScanExec}
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.execution.streaming._
+import org.apache.spark.sql.execution.streaming.operators.stateful.transformwithstate.TransformWithStateExec
 import org.apache.spark.sql.execution.streaming.sources.{ContinuousMemorySink, LowLatencyMemoryStream}
 import org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer
-import org.apache.spark.sql.streaming.{StreamingQuery, Trigger}
+import org.apache.spark.sql.streaming.{OutputMode, StatefulProcessor, StreamingQuery, TimeMode,
+  TimerValues, Trigger, TTLConfig, ValueState}
 import org.apache.spark.sql.streaming.OutputMode.Update
 import org.apache.spark.sql.streaming.util.GlobalSingletonManualClock
 import org.apache.spark.sql.test.TestSparkSession
 import org.apache.spark.util.SystemClock
 
+private class KafkaRunningCountStatefulProcessor
+  extends StatefulProcessor[String, String, (String, Long)] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState(
+      "count", Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[String],
+      timerValues: TimerValues): Iterator[(String, Long)] = {
+    inputRows.map { _ =>
+      val count = Option(countState.get()).getOrElse(0L) + 1L
+      countState.update(count)
+      (key, count)
+    }
+  }
+}
+
 class KafkaRealTimeModeSuite
   extends KafkaSourceTest
     with Matchers {
@@ -164,6 +191,97 @@ class KafkaRealTimeModeSuite
       WaitUntilCurrentBatchProcessed)
   }
 
+  test("transformWithState uses a pipelined shuffle and recovers RocksDB changelog state") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "2",
+        "spark.sql.streaming.stateStore.rocksdb.changelogCheckpointing.enabled" -> "true") {
+      val topic = newTopic()
+      testUtils.createTopic(topic, partitions = 2)
+      testUtils.sendMessages(topic, Array("a", "a"), Some(0))
+      testUtils.sendMessages(topic, Array("b"), Some(1))
+
+      val counts = spark.readStream
+        .format("kafka")
+        .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+        .option("subscribe", topic)
+        .option("startingOffsets", "earliest")
+        .load()
+        .selectExpr("CAST(value AS STRING)")
+        .as[String]
+        .groupByKey(identity)
+        .transformWithState(
+          new KafkaRunningCountStatefulProcessor,
+          TimeMode.None(),
+          Update)
+
+      testStream(counts, Update, sink = new ContinuousMemorySink())(
+        StartStream(),
+        CheckAnswerWithTimeout(60000, ("a", 1L), ("a", 2L), ("b", 1L)),
+        Execute { q =>
+          val plan = q.lastExecution.executedPlan
+          val statefulOperators = plan.collect { case t: TransformWithStateExec => t }
+          assert(statefulOperators.size == 1, plan)
+          assert(statefulOperators.head.isRealTimeMode, plan)
+
+          val exchanges = plan.collect { case s: ShuffleExchangeExec => s }
+          assert(exchanges.nonEmpty, plan)
+          assert(exchanges.forall(_.pipelined), plan)
+        },
+        WaitUntilCurrentBatchProcessed,
+        StopStream,
+        new ExternalAction() {
+          override def runAction(): Unit = {
+            testUtils.sendMessages(topic, Array("a"), Some(0))
+            testUtils.sendMessages(topic, Array("b", "b"), Some(1))
+          }
+        },
+        StartStream(),
+        CheckAnswerWithTimeout(
+          60000,
+          ("a", 1L), ("a", 2L), ("b", 1L),
+          ("a", 3L), ("b", 2L), ("b", 3L)),
+        WaitUntilCurrentBatchProcessed)
+    }
+  }
+
+  test("transformWithState remains in real-time mode when latestOffset returns null") {
+    val topic = newTopic()
+
+    val counts = spark.readStream
+      .format("kafka")
+      .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+      .option("kafka.metadata.max.age.ms", "1")
+      .option("subscribePattern", "^" + Pattern.quote(topic) + "$")
+      .option("startingOffsets", "earliest")
+      .load()
+      .selectExpr("CAST(value AS STRING)")
+      .as[String]
+      .groupByKey(identity)
+      .transformWithState(
+        new KafkaRunningCountStatefulProcessor,
+        TimeMode.None(),
+        Update)
+
+    testStream(counts, Update, sink = new ContinuousMemorySink())(
+      StartStream(),
+      WaitUntilBatchProcessed(0),
+      Execute { q =>
+        val plan = q.lastExecution.executedPlan
+        assert(plan.collect { case _: RealTimeStreamScanExec => true }.isEmpty, plan)
+        val statefulOperators = plan.collect { case t: TransformWithStateExec => t }
+        assert(statefulOperators.size == 1, plan)
+        assert(statefulOperators.head.isRealTimeMode, plan)
+      },
+      new ExternalAction() {
+        override def runAction(): Unit = {
+          testUtils.createTopic(topic, partitions = 1)
+          testUtils.sendMessages(topic, Array("a"))
+        }
+      },
+      CheckAnswerWithTimeout(60000, ("a", 1L)),
+      StopStream)
+  }
+
   // A simple unit test that reads from Kakfa source, does a simple map and writes to memory
   // sink. Make sure there is no data for a whole batch. Also, after restart the first batch
   // has no data.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
index 1e10767b53163..27f7599fa6536 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
@@ -657,6 +657,18 @@ object UnsupportedOperationChecker extends Logging {
     if (outputMode != InternalOutputModes.Update) {
       throwRealTimeError("OUTPUT_MODE_NOT_SUPPORTED", Map("outputMode" -> outputMode.toString))
     }
+
+    plan.foreachUp {
+      case u: Union =>
+        // Block stateful operators before union
+        u.foreachUp {
+          case statefulOp @ (_: Aggregate | _: TransformWithState |
+               _: TransformWithStateInPySpark | _: Deduplicate) if statefulOp.isStateful =>
+            throwRealTimeError("STATEFUL_OPERATORS_BEFORE_UNION_NOT_SUPPORTED", Map.empty)
+          case _ =>
+        }
+      case _ =>
+    }
   }
 
   private def throwRealTimeError(subClass: String, args: Map[String, String]): Unit = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 8803546e53f84..ff7c005e4a26c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -3627,13 +3627,12 @@ object SQLConf {
   val STREAMING_REAL_TIME_MODE_DANGEROUSLY_ALLOW_CHECKPOINT_V1 =
     buildConf("spark.sql.streaming.realTimeMode.dangerouslyAllowCheckpointV1.enabled")
       .internal()
-      .doc("Whether to allow a Real-Time Mode query to start on a checkpoint whose commit log " +
-        "is at version 1. Real-Time Mode re-executes a failed batch, and with checkpoint format " +
+      .doc("Whether to allow a Real-Time Mode query to start with state store checkpoint format " +
+        "version 1. Real-Time Mode re-executes a failed batch, and with checkpoint format " +
         "version 1 the re-execution can reuse the state file names of the partially-written " +
         "failed batch, so starting on a version 1 checkpoint exposes the query to data loss on " +
-        "failure. Format version 2 avoids this with per-batch state store checkpoint ids, which " +
-        "only a commit log at version 2 or above can persist. Escape hatch only; prefer a fresh " +
-        "checkpoint location.")
+        "failure. Format version 2 avoids this with per-batch state store checkpoint ids. " +
+        "Escape hatch only; prefer a fresh checkpoint location.")
       .version("4.3.0")
       .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
       .booleanConf
@@ -4056,6 +4055,16 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
+  val STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS = buildConf(
+    "spark.sql.streaming.realTimeMode.transformWithState.ttlEvictionIntervalMs")
+    .internal()
+    .doc("The threshold in milliseconds to perform eviction of TTL when using the JVM " +
+      "transformWithState operator with real-time mode.")
+    .version("4.3.0")
+    .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+    .longConf
+    .createWithDefault(1 * 1000)
+
   val STREAMING_ASYNC_PROGRESS_TRACKING_REAL_TIME_MODE_ENABLED_BY_DEFAULT = buildConf(
     "spark.sql.streaming.realTimeMode.asyncProgressTrackingByDefault.enabled")
     .internal()
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
index 293523b86f998..6b0b8abdcf15b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
@@ -30,8 +30,8 @@ import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes._
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode}
-import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder}
+import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode, StatefulProcessor, TimeMode, TimerValues}
+import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StructType}
 
 /** A dummy command for testing unsupported operations. */
 case class DummyCommand() extends LeafCommand
@@ -967,6 +967,58 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
     )
   }
 
+  assertSupportedForRealTime(
+    "real-time with Scala transformWithState - update mode",
+    scalaTransformWithState(streamRelation),
+    Update
+  )
+
+  assertNotSupportedForRealTime(
+    "real-time with Scala transformWithState on both sides of union - update mode",
+    scalaTransformWithState(streamRelation)
+      .union(scalaTransformWithState(new TestStreamingRelation(attribute.newInstance()))),
+    Update,
+    "STREAMING_REAL_TIME_MODE.STATEFUL_OPERATORS_BEFORE_UNION_NOT_SUPPORTED"
+  )
+
+  assertSupportedForRealTime(
+    "real-time with batch aggregate before union - update mode",
+    streamRelation
+      .join(Aggregate(Nil, aggExprs("c"), batchRelation), joinType = Inner)
+      .select(attribute)
+      .union(new TestStreamingRelation(attribute.newInstance())),
+    Update
+  )
+
+  private def scalaTransformWithState(child: LogicalPlan): TransformWithState = {
+    val statefulProcessor = new StatefulProcessor[Any, Any, Any] {
+      override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {}
+
+      override def handleInputRows(
+          key: Any,
+          inputRows: Iterator[Any],
+          timerValues: TimerValues): Iterator[Any] = Iterator.empty
+    }
+    val keyEncoder = ExpressionEncoder(new StructType().add("a", IntegerType))
+      .asInstanceOf[ExpressionEncoder[Any]]
+    new TransformWithState(
+      keyDeserializer = attribute,
+      valueDeserializer = attribute,
+      groupingAttributes = Seq(attribute),
+      dataAttributes = Seq(attribute),
+      statefulProcessor = statefulProcessor,
+      timeMode = NoTime,
+      outputMode = Update,
+      keyEncoder = keyEncoder,
+      outputObjAttr = attribute,
+      child = child,
+      hasInitialState = false,
+      initialStateGroupingAttrs = Seq(attribute),
+      initialStateDataAttrs = Seq(attribute),
+      initialStateDeserializer = attribute,
+      initialState = LocalRelation(Seq.empty[Attribute]))
+  }
+
   /*
     =======================================================================================
                                      TESTING FUNCTIONS
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 577f09d9a685b..42c1121f85dac 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -86,6 +86,19 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
     }
   }
 
+  /**
+   * Whether this plan reads a streaming source in Real-Time Mode, which is the case when the
+   * relation carries a real-time mode duration -- the same signal that decides whether to plan
+   * a [[org.apache.spark.sql.execution.datasources.v2.RealTimeStreamScanExec]] for it.
+   */
+  private def isRealTimeMode(plan: LogicalPlan): Boolean = {
+    plan.collectLeaves().exists {
+      case s: StreamingDataSourceV2ScanRelation =>
+        s.relation.realTimeModeDuration.isDefined
+      case _ => false
+    }
+  }
+
   /**
    * Plans special cases of limit operators.
    */
@@ -560,20 +573,6 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
    * [[org.apache.spark.sql.execution.streaming.StreamExecution]]
    */
   object StatefulAggregationStrategy extends Strategy {
-
-    /**
-     * Whether this plan reads a streaming source in Real-Time Mode, which is the case when the
-     * relation carries a real-time mode duration -- the same signal that decides whether to plan
-     * a [[org.apache.spark.sql.execution.datasources.v2.RealTimeStreamScanExec]] for it.
-     */
-    private def isRealTimeMode(plan: LogicalPlan): Boolean = {
-      plan.collectLeaves().exists {
-        case s: StreamingDataSourceV2ScanRelation =>
-          s.relation.realTimeModeDuration.isDefined
-        case _ => false
-      }
-    }
-
     override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
       case _ if !plan.isStreaming => Nil
 
@@ -928,6 +927,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
           eventTimeWatermarkForEviction = None,
           planLater(child),
           isStreaming = true,
+          isRealTimeMode = isRealTimeMode(plan),
           hasInitialState,
           initialStateGroupingAttrs,
           initialStateDataAttrs,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/statefulOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/statefulOperators.scala
index 48d08b719492e..5fb00fbb2a324 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/statefulOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/statefulOperators.scala
@@ -1626,7 +1626,8 @@ trait SchemaValidationUtils extends Logging {
       stateSchemaDir: Path,
       session: SparkSession,
       operatorStateMetadataVersion: Int = 2,
-      stateStoreEncodingFormat: String = StateStoreEncoding.UnsafeRow.toString
+      stateStoreEncodingFormat: String = StateStoreEncoding.UnsafeRow.toString,
+      isRealTimeMode: Boolean = false
   ): List[StateSchemaValidationResult] = {
     assert(stateSchemaVersion >= 3)
     val usingAvro = stateStoreEncodingFormat == StateStoreEncoding.Avro.toString
@@ -1634,8 +1635,11 @@ trait SchemaValidationUtils extends Logging {
     val newStateSchemaFilePath =
       new Path(stateSchemaDir, s"${batchId}_${UUID.randomUUID().toString}")
     val metadataPath = new Path(info.checkpointLocation, s"${info.operatorId}")
+    // RTM can leave metadata for an uncommitted attempt of the current batch, so only metadata
+    // from the previous committed batch is safe to use for schema validation.
+    val metadataBatchId = if (isRealTimeMode) batchId - 1 else batchId
     val metadataReader = OperatorStateMetadataReader.createReader(
-      metadataPath, hadoopConf, operatorStateMetadataVersion, batchId)
+      metadataPath, hadoopConf, operatorStateMetadataVersion, metadataBatchId)
     val operatorStateMetadata = try {
       metadataReader.read()
     } catch {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExec.scala
index f0e3003b2b710..1966347ea8571 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExec.scala
@@ -27,8 +27,10 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.WidenStatefulOpNullability
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, UnsafeRow}
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.datasources.v2.LowLatencyClock
 import org.apache.spark.sql.execution.streaming.operators.stateful.{StatefulOperatorStateInfo, StatefulOperatorsUtils}
 import org.apache.spark.sql.execution.streaming.operators.stateful.join.StreamingSymmetricHashJoinHelper.StateStoreAwareZipPartitionsHelper
 import org.apache.spark.sql.execution.streaming.operators.stateful.transformwithstate.statefulprocessor.{DriverStatefulProcessorHandleImpl, ImplicitGroupingKeyTracker, StatefulProcessorHandleImpl, StatefulProcessorHandleState}
@@ -55,6 +57,7 @@ import org.apache.spark.util.{CompletionIterator, SerializableConfiguration, Uti
  * @param eventTimeWatermarkForLateEvents event time watermark for filtering late events
  * @param eventTimeWatermarkForEviction event time watermark for state eviction
  * @param isStreaming defines whether the query is streaming or batch
+ * @param isRealTimeMode defines whether the query is running in Real-Time Mode
  * @param child the physical plan for the underlying data
  */
 case class TransformWithStateExec(
@@ -74,6 +77,7 @@ case class TransformWithStateExec(
     eventTimeWatermarkForEviction: Option[Long],
     child: SparkPlan,
     isStreaming: Boolean = true,
+    isRealTimeMode: Boolean = false,
     hasInitialState: Boolean = false,
     initialStateGroupingAttrs: Seq[Attribute],
     initialStateDataAttrs: Seq[Attribute],
@@ -87,7 +91,8 @@ case class TransformWithStateExec(
     eventTimeWatermarkForEviction,
     child,
     initialStateGroupingAttrs,
-    initialState)
+    initialState,
+    isRealTimeMode)
   with ObjectProducerExec {
 
   override def output: Seq[Attribute] =
@@ -200,7 +205,10 @@ case class TransformWithStateExec(
     }
   }
 
-  private def handleInputRows(keyRow: UnsafeRow, valueRowIter: Iterator[InternalRow]):
+  private def handleInputRows(
+      keyRow: UnsafeRow,
+      currentProcessingTimeMs: Option[Long],
+      valueRowIter: Iterator[InternalRow]):
     Iterator[InternalRow] = {
 
     val getOutputRow = ObjectOperator.wrapObjectToRow(outputObjectType)
@@ -218,7 +226,7 @@ case class TransformWithStateExec(
      statefulProcessor.handleInputRows(
         keyObj,
         valueObjIter,
-        new TimerValuesImpl(batchTimestampMs, eventTimeWatermarkForEviction)).map { obj =>
+        new TimerValuesImpl(currentProcessingTimeMs, eventTimeWatermarkForEviction)).map { obj =>
         getOutputRow(obj)
       }
     }
@@ -254,20 +262,86 @@ case class TransformWithStateExec(
     val groupedIter = GroupedIterator(dataIter, groupingAttributes, child.output)
     groupedIter.flatMap { case (keyRow, valueRowIter) =>
       val keyUnsafeRow = keyRow.asInstanceOf[UnsafeRow]
-      handleInputRows(keyUnsafeRow, valueRowIter)
+      handleInputRows(keyUnsafeRow, batchTimestampMs, valueRowIter)
+    }
+  }
+
+  // This is used in Real-time mode to process the data and expire timers interleaved; we also
+  // can do incremental cleanup for every record that we process.
+  private def processNewDataAndTimersAndTTL(
+      dataIter: Iterator[InternalRow],
+      processorHandle: StatefulProcessorHandleImpl): Iterator[InternalRow] = {
+    val keyProjection = GenerateUnsafeProjection.generate(groupingAttributes, child.output)
+    val ttlEvictionIntervalMs = conf.getConf(
+      SQLConf.STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS)
+    var lastTTLEvictionTriggeredMillis = -1L
+
+    dataIter.flatMap { row =>
+      val keyRow = keyProjection(row)
+      val valueRowIter = watermarkPredicateForDataForLateEvents match {
+        case Some(predicate) if timeMode == TimeMode.EventTime() =>
+          applyRemovingRowsOlderThanWatermark(Iterator.single(row), predicate)
+        case _ =>
+          Iterator.single(row)
+      }
+
+      val outputDataItr = if (valueRowIter.isEmpty) {
+        Iterator.empty
+      } else {
+        handleInputRows(
+          keyRow, Some(LowLatencyClock.getClock.getTimeMillis()), valueRowIter)
+      }
+
+      // handle expired timer rows
+      // late bind for the expired timers to deal with lazy iterators
+      val expiredTimersOutputItr = new Iterator[InternalRow] {
+        private lazy val itr = getIterator()
+        override def hasNext: Boolean = itr.hasNext
+        override def next(): InternalRow = itr.next()
+        private def getIterator(): Iterator[InternalRow] = {
+          processTimers(
+            timeMode,
+            Some(LowLatencyClock.getClock.getTimeMillis()),
+            processorHandle,
+            useReusableIterator = true)
+        }
+      }
+
+      // handle expired state values via ttl cleanup
+      val cleanupTtlIter = new Iterator[InternalRow] {
+        private var yetEvaluated = true
+
+        override def hasNext: Boolean = {
+          if (yetEvaluated) {
+            yetEvaluated = false
+            val currentTimeMs = LowLatencyClock.getClock.getTimeMillis()
+            if (currentTimeMs - lastTTLEvictionTriggeredMillis > ttlEvictionIntervalMs) {
+              processorHandle.doTtlCleanup(currentTimeMs)
+              lastTTLEvictionTriggeredMillis = currentTimeMs
+            }
+          }
+          false
+        }
+
+        override def next(): InternalRow =
+          throw new IllegalStateException("next() should not be called on this iterator")
+      }
+
+      outputDataItr ++ expiredTimersOutputItr ++ cleanupTtlIter
     }
   }
 
   private def handleTimerRows(
       keyObj: Any,
       expiryTimestampMs: Long,
+      currentProcessingTimeMs: Option[Long],
       processorHandle: StatefulProcessorHandleImpl): Iterator[InternalRow] = {
     val getOutputRow = ObjectOperator.wrapObjectToRow(outputObjectType)
     ImplicitGroupingKeyTracker.setImplicitKey(keyObj)
     val mappedIterator = withStatefulProcessorErrorHandling("handleExpiredTimer") {
       statefulProcessor.handleExpiredTimer(
         keyObj,
-        new TimerValuesImpl(batchTimestampMs, eventTimeWatermarkForEviction),
+        new TimerValuesImpl(currentProcessingTimeMs, eventTimeWatermarkForEviction),
         new ExpiredTimerInfoImpl(Some(expiryTimestampMs))).map { obj =>
         getOutputRow(obj)
       }
@@ -281,31 +355,37 @@ case class TransformWithStateExec(
 
   private def processTimers(
       timeMode: TimeMode,
-      processorHandle: StatefulProcessorHandleImpl): Iterator[InternalRow] = {
+      currentProcessingTimeMs: Option[Long],
+      processorHandle: StatefulProcessorHandleImpl,
+      useReusableIterator: Boolean = false): Iterator[InternalRow] = {
     val numExpiredTimers = longMetric("numExpiredTimers")
-    // SPARK-56566: Timers are always scanned without a lower bound (full scan up to the current
-    // batch timestamp / eviction watermark). We intentionally do not pass
-    // prevBatchTimestampMs / lateEventsWatermark as the exclusive lower bound here:
-    // registerTimer has no guard on the registered expiry, so a user-registered timer with expiry
-    // at or below the previous batch's lower bound would be silently dropped by a bounded scan.
-    // Revisit once registerTimer enforces ts > currentBatchTimestamp / watermark.
+    def getExpiredTimers(expiryTimestampMs: Long): Iterator[(Any, Long)] = {
+      if (useReusableIterator) {
+        processorHandle.getExpiredTimersReusableIterator(expiryTimestampMs)
+      } else {
+        processorHandle.getExpiredTimers(expiryTimestampMs)
+      }
+    }
+    // The final batch scan has no lower bound. RTM's reusable per-row scan resumes from its
+    // previous expiration threshold to avoid repeatedly scanning older timer entries.
     timeMode match {
       case ProcessingTime =>
-        assert(batchTimestampMs.isDefined)
-        val batchTimestamp = batchTimestampMs.get
-        processorHandle.getExpiredTimers(batchTimestamp)
+        assert(currentProcessingTimeMs.isDefined)
+        getExpiredTimers(currentProcessingTimeMs.get)
           .flatMap { case (keyObj, expiryTimestampMs) =>
             numExpiredTimers += 1
-            handleTimerRows(keyObj, expiryTimestampMs, processorHandle)
+            handleTimerRows(
+              keyObj, expiryTimestampMs, currentProcessingTimeMs, processorHandle)
           }
 
       case EventTime =>
         assert(eventTimeWatermarkForEviction.isDefined)
         val watermark = eventTimeWatermarkForEviction.get
-        processorHandle.getExpiredTimers(watermark)
+        getExpiredTimers(watermark)
           .flatMap { case (keyObj, expiryTimestampMs) =>
             numExpiredTimers += 1
-            handleTimerRows(keyObj, expiryTimestampMs, processorHandle)
+            handleTimerRows(
+              keyObj, expiryTimestampMs, currentProcessingTimeMs, processorHandle)
           }
 
       case _ => Iterator.empty
@@ -334,17 +414,21 @@ case class TransformWithStateExec(
     val updatesStartTimeNs = currentTimeNs
     var timerProcessingStartTimeNs = currentTimeNs
 
-    // If timeout is based on event time, then filter late data based on watermark
-    val filteredIter = watermarkPredicateForDataForLateEvents match {
-      case Some(predicate) if timeMode == TimeMode.EventTime() =>
-        applyRemovingRowsOlderThanWatermark(iter, predicate)
-      case _ =>
-        iter
+    val dataOutputIter = if (isRealTimeMode) {
+      processNewDataAndTimersAndTTL(iter, processorHandle)
+    } else {
+      // If timeout is based on event time, then filter late data based on watermark.
+      val filteredIter = watermarkPredicateForDataForLateEvents match {
+        case Some(predicate) if timeMode == TimeMode.EventTime() =>
+          applyRemovingRowsOlderThanWatermark(iter, predicate)
+        case _ =>
+          iter
+      }
+      processNewData(filteredIter)
     }
 
-    val newDataProcessorIter =
-      CompletionIterator[InternalRow, Iterator[InternalRow]](
-      processNewData(filteredIter), {
+    val newDataProcessorIter = CompletionIterator[InternalRow, Iterator[InternalRow]](
+      dataOutputIter, {
         // Note: Due to the iterator lazy execution, this metric also captures the time taken
         // by the upstream (consumer) operators in addition to the processing in this operator.
         allUpdatesTimeMs += NANOSECONDS.toMillis(System.nanoTime - updatesStartTimeNs)
@@ -364,7 +448,14 @@ case class TransformWithStateExec(
       override def next() = itr.next()
       private def getIterator(): Iterator[InternalRow] =
         CompletionIterator[InternalRow, Iterator[InternalRow]](
-          processTimers(timeMode, processorHandle), {
+          processTimers(
+            timeMode,
+            if (isRealTimeMode) {
+              Some(LowLatencyClock.getClock.getTimeMillis())
+            } else {
+              batchTimestampMs
+            },
+            processorHandle), {
           // Note: `timerProcessingTimeMs` also includes the time the parent operators take for
           // processing output returned from the timers that fire.
           timerProcessingTimeMs +=
@@ -409,7 +500,8 @@ case class TransformWithStateExec(
     val info = getStateInfo
     val stateSchemaDir = stateSchemaDirPath()
     validateAndWriteStateSchema(hadoopConf, batchId, stateSchemaVersion,
-      info, stateSchemaDir, session, operatorStateMetadataVersion, conf.stateStoreEncodingFormat)
+      info, stateSchemaDir, session, operatorStateMetadataVersion, conf.stateStoreEncodingFormat,
+      isRealTimeMode = isRealTimeMode)
   }
 
   override protected def doExecute(): RDD[InternalRow] = {
@@ -533,9 +625,10 @@ case class TransformWithStateExec(
    */
   private def processData(store: StateStore, singleIterator: Iterator[InternalRow]):
     CompletionIterator[InternalRow, Iterator[InternalRow]] = {
+    val currentTimestampMs = if (isRealTimeMode) Some(currentTimestampMsFn) else None
     val processorHandle = new StatefulProcessorHandleImpl(
       store, getStateInfo.queryRunId, keyEncoder, timeMode,
-      isStreaming, batchTimestampMs, prevBatchTimestampMs, metrics)
+      isStreaming, batchTimestampMs, prevBatchTimestampMs, metrics, currentTimestampMs)
     assert(processorHandle.getHandleState == StatefulProcessorHandleState.CREATED)
     statefulProcessor.setHandle(processorHandle)
     withStatefulProcessorErrorHandling("init") {
@@ -550,8 +643,10 @@ case class TransformWithStateExec(
       childDataIterator: Iterator[InternalRow],
       initStateIterator: Iterator[InternalRow]):
     CompletionIterator[InternalRow, Iterator[InternalRow]] = {
+    val currentTimestampMs = if (isRealTimeMode) Some(currentTimestampMsFn) else None
     val processorHandle = new StatefulProcessorHandleImpl(store, getStateInfo.queryRunId,
-      keyEncoder, timeMode, isStreaming, batchTimestampMs, prevBatchTimestampMs, metrics)
+      keyEncoder, timeMode, isStreaming, batchTimestampMs, prevBatchTimestampMs, metrics,
+      currentTimestampMs)
     assert(processorHandle.getHandleState == StatefulProcessorHandleState.CREATED)
     statefulProcessor.setHandle(processorHandle)
     withStatefulProcessorErrorHandling("init") {
@@ -626,6 +721,7 @@ object TransformWithStateExec {
       None,
       child,
       isStreaming = false,
+      isRealTimeMode = false,
       hasInitialState,
       initialStateGroupingAttrs,
       initialStateDataAttrs,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExecBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExecBase.scala
index f5abe333d0f35..82fce42870d4a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExecBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/TransformWithStateExecBase.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, SortOrde
 import org.apache.spark.sql.catalyst.plans.logical.{EventTime, ProcessingTime}
 import org.apache.spark.sql.catalyst.plans.physical.Distribution
 import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan}
+import org.apache.spark.sql.execution.datasources.v2.LowLatencyClock
 import org.apache.spark.sql.execution.streaming.operators.stateful.{StatefulOperatorCustomMetric, StatefulOperatorCustomSumMetric, StatefulOperatorPartitioning, StateStoreWriter, WatermarkSupport}
 import org.apache.spark.sql.execution.streaming.operators.stateful.transformwithstate.statefulprocessor.ImplicitGroupingKeyTracker
 import org.apache.spark.sql.execution.streaming.state.{OperatorStateMetadata, RocksDBStateStoreProvider, StateStoreErrors, TransformWithStateUserFunctionException}
@@ -46,7 +47,8 @@ abstract class TransformWithStateExecBase(
     eventTimeWatermarkForEviction: Option[Long],
     child: SparkPlan,
     initialStateGroupingAttrs: Seq[Attribute],
-    initialState: SparkPlan)
+    initialState: SparkPlan,
+    isRealTimeMode: Boolean = false)
   extends BinaryExecNode
   with StateStoreWriter
   with WatermarkSupport
@@ -69,6 +71,22 @@ abstract class TransformWithStateExecBase(
   // The keys that may have a watermark attribute.
   override def keyExpressions: Seq[Attribute] = groupingAttributes
 
+  @inline
+  protected lazy val currentTimestampMsFn: () => Long = () => {
+    assert(
+      batchTimestampMs.isDefined,
+      "batchTimestampMs should be set when " +
+        "invoking the currentTimestampMs function. This function must have been " +
+        "eagerly created; ensure it is lazy and never invoked before physical planning.")
+    // In real-time mode, we use the local wall time as the current processing time.
+    // Skew between executors is possible, but we assume it is acceptable for now.
+    if (isRealTimeMode) {
+      LowLatencyClock.getClock.getTimeMillis()
+    } else {
+      batchTimestampMs.get
+    }
+  }
+
   /**
    * Distribute by grouping attributes - We need the underlying data and the initial state data to
    * have the same grouping so that the data are co-located on the same task.
@@ -89,9 +107,18 @@ abstract class TransformWithStateExecBase(
    * We need the initial state to also use the ordering as the data so that we can co-locate the
    * keys from the underlying data and the initial state.
    */
-  override def requiredChildOrdering: Seq[Seq[SortOrder]] = Seq(
-    groupingAttributes.map(SortOrder(_, Ascending)),
-    initialStateGroupingAttrs.map(SortOrder(_, Ascending)))
+  override def requiredChildOrdering: Seq[Seq[SortOrder]] = {
+    if (isRealTimeMode) {
+      // In real-time mode, we don't need to order the initial state data since we produce the
+      // (key, value) pair for every single data. Also, streaming shuffle does not support
+      // sorting by nature.
+      Seq.fill(children.size)(Nil)
+    } else {
+      Seq(
+        groupingAttributes.map(SortOrder(_, Ascending)),
+        initialStateGroupingAttrs.map(SortOrder(_, Ascending)))
+    }
+  }
 
   override def shouldRunAnotherBatch(newInputWatermark: Long): Boolean = {
     if (timeMode == ProcessingTime) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/statefulprocessor/StatefulProcessorHandleImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/statefulprocessor/StatefulProcessorHandleImpl.scala
index 291cc02ea989b..25bdc58262faf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/statefulprocessor/StatefulProcessorHandleImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/statefulprocessor/StatefulProcessorHandleImpl.scala
@@ -106,6 +106,8 @@ class QueryInfoImpl(
  * @param isStreaming - defines whether the query is streaming or batch
  * @param batchTimestampMs - timestamp for the current batch if available
  * @param metrics - metrics to be updated as part of stateful processing
+ * @param currentTimestampMs - optional function for obtaining the current processing time. Real-
+ *                             time mode supplies a live clock; other modes use batchTimestampMs.
  */
 class StatefulProcessorHandleImpl(
     store: StateStore,
@@ -115,7 +117,8 @@ class StatefulProcessorHandleImpl(
     isStreaming: Boolean = true,
     batchTimestampMs: Option[Long] = None,
     prevBatchTimestampMs: Option[Long] = None,
-    metrics: Map[String, SQLMetric] = Map.empty)
+    metrics: Map[String, SQLMetric] = Map.empty,
+    currentTimestampMs: Option[() => Long] = None)
   extends StatefulProcessorHandleImplBase(timeMode, keyEncoder) with Logging {
   import StatefulProcessorHandleState._
 
@@ -125,6 +128,10 @@ class StatefulProcessorHandleImpl(
    */
   private[sql] val ttlStates: util.List[TTLState] = new util.ArrayList[TTLState]()
 
+  private lazy val ttlTimestampMs = currentTimestampMs.orElse {
+    batchTimestampMs.map(timestamp => () => timestamp)
+  }
+
   private val BATCH_QUERY_ID = "00000000-0000-0000-0000-000000000000"
 
   currState = CREATED
@@ -187,6 +194,15 @@ class StatefulProcessorHandleImpl(
     timerState.getExpiredTimers(expiryTimestampMs, prevExpiryTimestampMs)
   }
 
+  /**
+   * Return expired timers through a cached native iterator. RTM invokes this once per input row,
+   * so reusing the iterator avoids allocating native scan resources for every row.
+   */
+  def getExpiredTimersReusableIterator(expiryTimestampMs: Long): Iterator[(Any, Long)] = {
+    verifyTimerOperations("get_expired_timers")
+    timerState.getExpiredTimersReusable(expiryTimestampMs)
+  }
+
   /**
    * Function to list all the registered timers for given implicit key
    * Note: calling listTimers() within the `handleInputRows` method of the StatefulProcessor
@@ -204,9 +220,13 @@ class StatefulProcessorHandleImpl(
    * which is expired will be cleaned up from StateStore.
    */
   def doTtlCleanup(): Unit = {
+    ttlTimestampMs.foreach(currentTimestampMs => doTtlCleanup(currentTimestampMs()))
+  }
+
+  def doTtlCleanup(evictionTimestampMs: Long): Unit = {
     val numValuesRemovedDueToTTLExpiry = metrics.get("numValuesRemovedDueToTTLExpiry").get
     ttlStates.forEach { s =>
-      numValuesRemovedDueToTTLExpiry += s.clearExpiredStateForAllKeys()
+      numValuesRemovedDueToTTLExpiry += s.clearExpiredStateForAllKeys(evictionTimestampMs)
     }
   }
 
@@ -242,9 +262,9 @@ class StatefulProcessorHandleImpl(
     val stateEncoder = encoderFor[T].asInstanceOf[ExpressionEncoder[Any]]
     val result = if (ttlEnabled) {
       validateTTLConfig(ttlConfig, stateName)
-      assert(batchTimestampMs.isDefined)
+      assert(ttlTimestampMs.isDefined)
       val valueStateWithTTL = new ValueStateImplWithTTL[T](store, stateName,
-        keyEncoder, stateEncoder, ttlConfig, batchTimestampMs.get,
+        keyEncoder, stateEncoder, ttlConfig, ttlTimestampMs.get,
         prevBatchTimestampMs, metrics)
       ttlStates.add(valueStateWithTTL)
       TWSMetricsUtils.incrementMetric(metrics, "numValueStateWithTTLVars")
@@ -292,9 +312,9 @@ class StatefulProcessorHandleImpl(
     val stateEncoder = encoderFor[T].asInstanceOf[ExpressionEncoder[Any]]
     val result = if (ttlEnabled) {
       validateTTLConfig(ttlConfig, stateName)
-      assert(batchTimestampMs.isDefined)
+      assert(ttlTimestampMs.isDefined)
       val listStateWithTTL = new ListStateImplWithTTL[T](store, stateName,
-        keyEncoder, stateEncoder, ttlConfig, batchTimestampMs.get,
+        keyEncoder, stateEncoder, ttlConfig, ttlTimestampMs.get,
         prevBatchTimestampMs, metrics)
       TWSMetricsUtils.incrementMetric(metrics, "numListStateWithTTLVars")
       ttlStates.add(listStateWithTTL)
@@ -331,9 +351,9 @@ class StatefulProcessorHandleImpl(
     val valEncoder = encoderFor[V].asInstanceOf[ExpressionEncoder[Any]]
     val result = if (ttlEnabled) {
       validateTTLConfig(ttlConfig, stateName)
-      assert(batchTimestampMs.isDefined)
+      assert(ttlTimestampMs.isDefined)
       val mapStateWithTTL = new MapStateImplWithTTL[K, V](store, stateName, keyEncoder, userKeyEnc,
-        valEncoder, ttlConfig, batchTimestampMs.get,
+        valEncoder, ttlConfig, ttlTimestampMs.get,
         prevBatchTimestampMs, metrics)
       TWSMetricsUtils.incrementMetric(metrics, "numMapStateWithTTLVars")
       ttlStates.add(mapStateWithTTL)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/timers/TimerStateImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/timers/TimerStateImpl.scala
index 977e15e38e66a..c93f434359368 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/timers/TimerStateImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/timers/TimerStateImpl.scala
@@ -16,6 +16,8 @@
  */
 package org.apache.spark.sql.execution.streaming.operators.stateful.transformwithstate.timers
 
+import java.io.Closeable
+
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.LogKeys.{EXPIRY_TIMESTAMP, KEY}
 import org.apache.spark.sql.catalyst.InternalRow
@@ -114,6 +116,15 @@ class TimerStateImpl(
 
   private val secIndexProjection = UnsafeProjection.create(keySchemaForSecIndex)
 
+  private var reusableIterator: Option[ReusableIterator[UnsafeRowPair]] = None
+  private var lastScannedExpiryTimestampMs = 0L
+
+  private lazy val expiryTimestampProjection = UnsafeProjection.create(
+    new StructType().add("expiryTimestampMs", LongType, nullable = false))
+
+  private lazy val lastScannedExpiryTimestampRow =
+    expiryTimestampProjection.apply(InternalRow(lastScannedExpiryTimestampMs))
+
   // Placeholder grouping-key struct used in range-scan boundary rows; see
   // [[RangeScanBoundaryUtils]] for rationale. Correctness relies on real stored
   // entries never having a null grouping-key struct, which is preserved by
@@ -236,6 +247,37 @@ class TimerStateImpl(
     val endKey = encodeTimestampAsKey(expiryTimestampMs)
     val iter = store.rangeScan(startKey, endKey, tsToKeyCFName)
 
+    getExpiredTimersIterator(iter, expiryTimestampMs, isReusable = false)
+  }
+
+  /**
+   * Return expired timers using one cached native iterator. After the first scan, refresh and
+   * resume from the previous scan's expiration threshold.
+   */
+  private[sql] def getExpiredTimersReusable(
+      expiryTimestampMs: Long): Iterator[(Any, Long)] = {
+    store match {
+      case reusableStore: SupportsReusableIterator =>
+        val iter = reusableIterator match {
+          case Some(existingIterator) =>
+            lastScannedExpiryTimestampRow.setLong(0, lastScannedExpiryTimestampMs)
+            existingIterator.refreshAndSeekToPrefix(lastScannedExpiryTimestampRow)
+            existingIterator
+          case None =>
+            val newIterator = reusableStore.reusableIterator(tsToKeyCFName)
+            reusableIterator = Some(newIterator)
+            newIterator
+        }
+        lastScannedExpiryTimestampMs = expiryTimestampMs
+        getExpiredTimersIterator(iter, expiryTimestampMs, isReusable = true)
+      case _ => getExpiredTimers(expiryTimestampMs)
+    }
+  }
+
+  private def getExpiredTimersIterator(
+      iter: Iterator[UnsafeRowPair] with Closeable,
+      expiryTimestampMs: Long,
+      isReusable: Boolean): Iterator[(Any, Long)] = {
     new NextIterator[(Any, Long)] {
       override protected def getNext(): (Any, Long) = {
         if (iter.hasNext) {
@@ -255,7 +297,9 @@ class TimerStateImpl(
       }
 
       override protected def close(): Unit = {
-        iter.close()
+        if (!isReusable) {
+          iter.close()
+        }
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ListStateImplWithTTL.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ListStateImplWithTTL.scala
index 10ec3a58500af..8c69dd7f33816 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ListStateImplWithTTL.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ListStateImplWithTTL.scala
@@ -34,7 +34,7 @@ import org.apache.spark.util.NextIterator
  * @param keyExprEnc - Spark SQL encoder for key
  * @param valEncoder - Spark SQL encoder for value
  * @param ttlConfig  - TTL configuration for values  stored in this state
- * @param batchTimestampMs - current batch processing timestamp.
+ * @param currentTimestampMs - function to get the current processing time timestamp.
  * @param prevBatchTimestampMs - batch timestamp from the previous micro-batch (exclusive).
  *                               Entries with expiration at or below this timestamp are assumed
  *                               to have been already cleaned up and will be skipped during
@@ -48,11 +48,11 @@ class ListStateImplWithTTL[S](
     keyExprEnc: ExpressionEncoder[Any],
     valEncoder: ExpressionEncoder[Any],
     ttlConfig: TTLConfig,
-    batchTimestampMs: Long,
+    currentTimestampMs: () => Long,
     prevBatchTimestampMs: Option[Long] = None,
     metrics: Map[String, SQLMetric])
   extends OneToManyTTLState(
-    stateName, store, keyExprEnc.schema, ttlConfig, batchTimestampMs,
+    stateName, store, keyExprEnc.schema, ttlConfig, currentTimestampMs,
     prevBatchTimestampMs, metrics) with ListState[S] {
 
   private lazy val stateTypesEncoder = StateTypesEncoder(keyExprEnc, valEncoder,
@@ -83,7 +83,7 @@ class ListStateImplWithTTL[S](
 
       override protected def getNext(): S = {
         val iter = unsafeRowValuesIterator.dropWhile { row =>
-          stateTypesEncoder.isExpired(row, batchTimestampMs)
+          stateTypesEncoder.isExpired(row, currentTimestampMs())
         }
 
         if (iter.hasNext) {
@@ -167,7 +167,7 @@ class ListStateImplWithTTL[S](
     var newMinExpirationMsOpt: Option[Long] = None
     var isFirst = true
     unsafeRowValuesIterator.foreach { encodedValue =>
-      if (!stateTypesEncoder.isExpired(encodedValue, batchTimestampMs)) {
+      if (!stateTypesEncoder.isExpired(encodedValue, currentTimestampMs())) {
         if (isFirst) {
           isFirst = false
           store.put(elementKey, encodedValue, stateName)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/MapStateImplWithTTL.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/MapStateImplWithTTL.scala
index 03aa8aaa6ace2..6eb6801700ed1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/MapStateImplWithTTL.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/MapStateImplWithTTL.scala
@@ -35,7 +35,7 @@ import org.apache.spark.util.NextIterator
  * @param userKeyEnc  - Spark SQL encoder for the map key
  * @param valEncoder - SQL encoder for state variable
  * @param ttlConfig  - the ttl configuration (time to live duration etc.)
- * @param batchTimestampMs - current batch processing timestamp.
+ * @param currentTimestampMs - function to get the current processing time timestamp.
  * @param prevBatchTimestampMs - batch timestamp from the previous micro-batch (exclusive).
  *                               Entries with expiration at or below this timestamp are assumed
  *                               to have been already cleaned up and will be skipped during
@@ -52,12 +52,12 @@ class MapStateImplWithTTL[K, V](
     userKeyEnc: ExpressionEncoder[Any],
     valEncoder: ExpressionEncoder[Any],
     ttlConfig: TTLConfig,
-    batchTimestampMs: Long,
+    currentTimestampMs: () => Long,
     prevBatchTimestampMs: Option[Long] = None,
     metrics: Map[String, SQLMetric])
   extends OneToOneTTLState(
     stateName, store, getCompositeKeySchema(keyExprEnc.schema, userKeyEnc.schema), ttlConfig,
-    batchTimestampMs, prevBatchTimestampMs, metrics) with MapState[K, V] with Logging {
+    currentTimestampMs, prevBatchTimestampMs, metrics) with MapState[K, V] with Logging {
 
   private val stateTypesEncoder = new CompositeKeyStateEncoder(
     keyExprEnc, userKeyEnc, valEncoder, stateName, hasTtl = true)
@@ -84,7 +84,7 @@ class MapStateImplWithTTL[K, V](
     val retRow = store.get(encodedCompositeKey, stateName)
 
     if (retRow != null) {
-      if (!stateTypesEncoder.isExpired(retRow, batchTimestampMs)) {
+      if (!stateTypesEncoder.isExpired(retRow, currentTimestampMs())) {
         stateTypesEncoder.decodeValue(retRow).asInstanceOf[V]
       } else {
         null.asInstanceOf[V]
@@ -107,7 +107,7 @@ class MapStateImplWithTTL[K, V](
 
     val encodedCompositeKey = stateTypesEncoder.encodeCompositeKey(key)
     val ttlExpirationMs = StateTTL
-      .calculateExpirationTimeForDuration(ttlConfig.ttlDuration, batchTimestampMs)
+      .calculateExpirationTimeForDuration(ttlConfig.ttlDuration, currentTimestampMs())
     val encodedValue = stateTypesEncoder.encodeValue(value, ttlExpirationMs)
 
     updatePrimaryAndSecondaryIndices(encodedCompositeKey, encodedValue, ttlExpirationMs)
@@ -120,7 +120,7 @@ class MapStateImplWithTTL[K, V](
     new NextIterator[(K, V)] {
       override protected def getNext(): (K, V) = {
         val iter = unsafeRowPairIterator.dropWhile { rowPair =>
-          stateTypesEncoder.isExpired(rowPair.value, batchTimestampMs)
+          stateTypesEncoder.isExpired(rowPair.value, currentTimestampMs())
         }
         if (iter.hasNext) {
           val currentRowPair = iter.next()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/TTLState.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/TTLState.scala
index cab6aa9630e9c..470c72b39ad5e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/TTLState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/TTLState.scala
@@ -84,9 +84,9 @@ trait TTLState {
   // a map key.
   private[sql] def elementKeySchema: StructType
 
-  // The timestamp at which the batch is being processed. All state variables that have
-  // an expiration at or before this timestamp must be cleaned up.
-  private[sql] def batchTimestampMs: Long
+  // Returns the current processing-time timestamp. It is fixed to the journaled batch timestamp
+  // in micro-batch mode and reads the executor clock in Real-Time Mode.
+  private[sql] def currentTimestampMs: () => Long
 
   // The batch timestamp from the previous micro-batch, used to derive the startKey
   // for scan-based TTL eviction. Entries at or below prevBatchTimestampMs were already
@@ -125,7 +125,7 @@ trait TTLState {
     UnsafeProjection.create(Array[DataType](NullType)).apply(InternalRow.apply(null))
 
   private[sql] final def ttlExpirationMs = StateTTL
-    .calculateExpirationTimeForDuration(ttlConfig.ttlDuration, batchTimestampMs)
+    .calculateExpirationTimeForDuration(ttlConfig.ttlDuration, currentTimestampMs())
 
   store.createColFamilyIfAbsent(
     TTL_INDEX,
@@ -171,14 +171,14 @@ trait TTLState {
   }
 
   // Returns an Iterator over the keys in the TTL index that have expired. Uses a bounded
-  // range scan over [prevBatchTimestampMs+1, batchTimestampMs+1) to skip entries that
+  // range scan over [prevBatchTimestampMs+1, evictionTimestampMs+1) to skip entries that
   // were already evicted in previous batches.
   //
   // This method does not delete the keys from the TTL index; it is the responsibility of
   // the caller to do so.
   //
   // The schema of the UnsafeRow returned by this iterator is (expirationMs, elementKey).
-  private[sql] def ttlEvictionIterator(): Iterator[UnsafeRow] = {
+  private[sql] def ttlEvictionIterator(evictionTimestampMs: Long): Iterator[UnsafeRow] = {
     val startKey = prevBatchTimestampMs.flatMap { prevTs =>
       if (prevTs < Long.MaxValue) {
         Some(TTL_ENCODER.encodeTTLRow(prevTs + 1, DEFAULT_ELEMENT_KEY).copy())
@@ -186,8 +186,8 @@ trait TTLState {
         None
       }
     }
-    val endKey = if (batchTimestampMs < Long.MaxValue) {
-      Some(TTL_ENCODER.encodeTTLRow(batchTimestampMs + 1, DEFAULT_ELEMENT_KEY).copy())
+    val endKey = if (evictionTimestampMs < Long.MaxValue) {
+      Some(TTL_ENCODER.encodeTTLRow(evictionTimestampMs + 1, DEFAULT_ELEMENT_KEY).copy())
     } else {
       None
     }
@@ -198,7 +198,7 @@ trait TTLState {
     // Safety filter: keep only truly expired entries
     ttlIterator.takeWhile { kv =>
       val expirationMs = kv.key.getLong(0)
-      StateTTL.isExpired(expirationMs, batchTimestampMs)
+      StateTTL.isExpired(expirationMs, evictionTimestampMs)
     }.map(_.key)
   }
 
@@ -218,7 +218,7 @@ trait TTLState {
    *
    * @return number of values cleaned up.
    */
-  private[sql] def clearExpiredStateForAllKeys(): Long
+  private[sql] def clearExpiredStateForAllKeys(evictionTimestampMs: Long): Long
 
   /**
    * When a user calls clear() on a stateful variable, this method is invoked to
@@ -253,14 +253,14 @@ abstract class OneToOneTTLState(
     storeArg: StateStore,
     elementKeySchemaArg: StructType,
     ttlConfigArg: TTLConfig,
-    batchTimestampMsArg: Long,
+    currentTimestampMsArg: () => Long,
     prevBatchTimestampMsArg: Option[Long],
     metricsArg: Map[String, SQLMetric]) extends TTLState {
   override private[sql] def stateName: String = stateNameArg
   override private[sql] def store: StateStore = storeArg
   override private[sql] def elementKeySchema: StructType = elementKeySchemaArg
   override private[sql] def ttlConfig: TTLConfig = ttlConfigArg
-  override private[sql] def batchTimestampMs: Long = batchTimestampMsArg
+  override private[sql] def currentTimestampMs: () => Long = currentTimestampMsArg
   override private[sql] def prevBatchTimestampMs: Option[Long] = prevBatchTimestampMsArg
   override private[sql] def metrics: Map[String, SQLMetric] = metricsArg
 
@@ -310,10 +310,10 @@ abstract class OneToOneTTLState(
     }
   }
 
-  override private[sql] def clearExpiredStateForAllKeys(): Long = {
+  override private[sql] def clearExpiredStateForAllKeys(evictionTimestampMs: Long): Long = {
     var numValuesExpired = 0L
 
-    ttlEvictionIterator().foreach { ttlKey =>
+    ttlEvictionIterator(evictionTimestampMs).foreach { ttlKey =>
       // Delete from secondary index
       deleteFromTTLIndex(ttlKey)
       // Delete from primary index
@@ -372,14 +372,14 @@ abstract class OneToManyTTLState(
     storeArg: StateStore,
     elementKeySchemaArg: StructType,
     ttlConfigArg: TTLConfig,
-    batchTimestampMsArg: Long,
+    currentTimestampMsArg: () => Long,
     prevBatchTimestampMsArg: Option[Long],
     metricsArg: Map[String, SQLMetric]) extends TTLState {
   override private[sql] def stateName: String = stateNameArg
   override private[sql] def store: StateStore = storeArg
   override private[sql] def elementKeySchema: StructType = elementKeySchemaArg
   override private[sql] def ttlConfig: TTLConfig = ttlConfigArg
-  override private[sql] def batchTimestampMs: Long = batchTimestampMsArg
+  override private[sql] def currentTimestampMs: () => Long = currentTimestampMsArg
   override private[sql] def prevBatchTimestampMs: Option[Long] = prevBatchTimestampMsArg
   override private[sql] def metrics: Map[String, SQLMetric] = metricsArg
 
@@ -517,10 +517,10 @@ abstract class OneToManyTTLState(
   // Clears all the expired values for the given elementKey.
   protected def clearExpiredValues(elementKey: UnsafeRow): ValueExpirationResult
 
-  override private[sql] def clearExpiredStateForAllKeys(): Long = {
+  override private[sql] def clearExpiredStateForAllKeys(evictionTimestampMs: Long): Long = {
     var totalNumValuesExpired = 0L
 
-    ttlEvictionIterator().foreach { ttlKey =>
+    ttlEvictionIterator(evictionTimestampMs).foreach { ttlKey =>
       val ttlRow = toTTLRow(ttlKey)
       val elementKey = ttlRow.elementKey
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ValueStateImplWithTTL.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ValueStateImplWithTTL.scala
index 1559acf7222cf..7dce677df69ac 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ValueStateImplWithTTL.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/operators/stateful/transformwithstate/ttl/ValueStateImplWithTTL.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.streaming.{TTLConfig, ValueState}
  * @param keyExprEnc - Spark SQL encoder for key
  * @param valEncoder - Spark SQL encoder for value
  * @param ttlConfig  - TTL configuration for values  stored in this state
- * @param batchTimestampMs - current batch processing timestamp.
+ * @param currentTimestampMs - function to get the current processing time timestamp.
  * @param prevBatchTimestampMs - batch timestamp from the previous micro-batch (exclusive).
  *                               Entries with expiration at or below this timestamp are assumed
  *                               to have been already cleaned up and will be skipped during
@@ -46,11 +46,11 @@ class ValueStateImplWithTTL[S](
     keyExprEnc: ExpressionEncoder[Any],
     valEncoder: ExpressionEncoder[Any],
     ttlConfig: TTLConfig,
-    batchTimestampMs: Long,
+    currentTimestampMs: () => Long,
     prevBatchTimestampMs: Option[Long] = None,
     metrics: Map[String, SQLMetric] = Map.empty)
   extends OneToOneTTLState(
-    stateName, store, keyExprEnc.schema, ttlConfig, batchTimestampMs,
+    stateName, store, keyExprEnc.schema, ttlConfig, currentTimestampMs,
     prevBatchTimestampMs, metrics) with ValueState[S] {
 
   private val stateTypesEncoder =
@@ -80,7 +80,7 @@ class ValueStateImplWithTTL[S](
       // Getting the 0th ordinal of the struct using valEncoder
       val resState = stateTypesEncoder.decodeValue(retRow)
 
-      if (!stateTypesEncoder.isExpired(retRow, batchTimestampMs)) {
+      if (!stateTypesEncoder.isExpired(retRow, currentTimestampMs())) {
         resState.asInstanceOf[S]
       } else {
         null.asInstanceOf[S]
@@ -95,7 +95,7 @@ class ValueStateImplWithTTL[S](
     val encodedKey = stateTypesEncoder.encodeGroupingKey()
 
     val ttlExpirationMs = StateTTL
-      .calculateExpirationTimeForDuration(ttlConfig.ttlDuration, batchTimestampMs)
+      .calculateExpirationTimeForDuration(ttlConfig.ttlDuration, currentTimestampMs())
     val encodedValue = stateTypesEncoder.encodeValue(newState, ttlExpirationMs)
 
     updatePrimaryAndSecondaryIndices(encodedKey, encodedValue, ttlExpirationMs)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/IncrementalExecution.scala
index e46a4bf28690d..9efee984457dd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/IncrementalExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/IncrementalExecution.scala
@@ -47,7 +47,7 @@ import org.apache.spark.sql.execution.streaming.operators.stateful.flatmapgroups
 import org.apache.spark.sql.execution.streaming.operators.stateful.join.{StreamingSymmetricHashJoinExec, StreamingSymmetricHashJoinHelper}
 import org.apache.spark.sql.execution.streaming.operators.stateful.transformwithstate.TransformWithStateExec
 import org.apache.spark.sql.execution.streaming.sources.WriteToMicroBatchDataSourceV1
-import org.apache.spark.sql.execution.streaming.state.{OperatorStateMetadataReader, OperatorStateMetadataV1, OperatorStateMetadataV2, OperatorStateMetadataWriter, StateSchemaBroadcast, StateSchemaMetadata}
+import org.apache.spark.sql.execution.streaming.state.{OperatorStateMetadata, OperatorStateMetadataReader, OperatorStateMetadataV1, OperatorStateMetadataV2, OperatorStateMetadataWriter, StateSchemaBroadcast, StateSchemaMetadata}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.util.{SerializableConfiguration, Utils}
@@ -83,7 +83,8 @@ class IncrementalExecution(
     val stateSchemaMetadatas: MutableMap[Long, StateSchemaBroadcast] =
       MutableMap[Long, StateSchemaBroadcast](),
     mode: CommandExecutionMode.Value = CommandExecutionMode.ALL,
-    val isTerminatingTrigger: Boolean = false)
+    val isTerminatingTrigger: Boolean = false,
+    val isRealTimeMode: Boolean = false)
   extends QueryExecution(sparkSession, logicalPlan, mode = mode,
     shuffleCleanupModeOpt =
       Some(QueryExecution.determineShuffleCleanupMode(sparkSession.sessionState.conf)),
@@ -111,6 +112,12 @@ class IncrementalExecution(
 
   private lazy val hadoopConf = sparkSession.sessionState.newHadoopConf()
 
+  // Populated while state schemas are validated during planning. In micro-batch mode each entry
+  // is also written immediately, preserving the existing behavior. Real-Time Mode writes these
+  // entries only after its delayed offset log entry is durable.
+  private var stateStoreWritersWithMetadata:
+      Option[Seq[(StateStoreWriter, OperatorStateMetadata)]] = None
+
   private[sql] val numStateStores = OffsetSeqMetadata.readValueOpt(offsetSeqMetadata,
       SQLConf.STATEFUL_SHUFFLE_PARTITIONS_INTERNAL)
     .map(SQLConf.SHUFFLE_PARTITIONS.valueConverter)
@@ -284,12 +291,11 @@ class IncrementalExecution(
                 ssw.validateNewMetadata(oldMetadata, metadata)
               case None =>
             }
-            val metadataWriter = OperatorStateMetadataWriter.createWriter(
-              new Path(checkpointLocation, ssw.getStateInfo.operatorId.toString),
-              hadoopConf,
-              ssw.operatorStateMetadataVersion,
-              Some(currentBatchId))
-            metadataWriter.write(metadata)
+            stateStoreWritersWithMetadata = Some(
+              stateStoreWritersWithMetadata.getOrElse(Seq.empty) :+ (ssw -> metadata))
+            if (!isRealTimeMode) {
+              writeStateMetadata(ssw, metadata)
+            }
             if (ssw.supportsSchemaEvolution) {
               val stateSchemaMetadata = StateSchemaMetadata
                 .createStateSchemaMetadata(checkpointLocation, hadoopConf, stateSchemaList.head)
@@ -318,6 +324,26 @@ class IncrementalExecution(
     }
   }
 
+  private def writeStateMetadata(
+      stateStoreWriter: StateStoreWriter,
+      metadata: OperatorStateMetadata): Unit = {
+    val metadataWriter = OperatorStateMetadataWriter.createWriter(
+      new Path(checkpointLocation, stateStoreWriter.getStateInfo.operatorId.toString),
+      hadoopConf,
+      metadata.version,
+      Some(currentBatchId))
+    metadataWriter.write(metadata)
+  }
+
+  /** Write state metadata recorded during planning. Used after the delayed RTM offset WAL write. */
+  private[streaming] def writeRecordedStateMetadata(): Unit = {
+    assert(stateStoreWritersWithMetadata.isDefined,
+      "stateStoreWritersWithMetadata must be defined before writing state metadata")
+    stateStoreWritersWithMetadata.get.foreach { case (stateStoreWriter, metadata) =>
+      writeStateMetadata(stateStoreWriter, metadata)
+    }
+  }
+
   object StateOpIdRule extends SparkPlanPartialRule {
     override val rule: PartialFunction[SparkPlan, SparkPlan] = {
       case a: StatefulStreamlineAggregateExec =>
@@ -405,6 +431,7 @@ class IncrementalExecution(
           prevBatchTimestampMs = prevOffsetSeqMetadata.map(_.batchTimestampMs),
           eventTimeWatermarkForLateEvents = None,
           eventTimeWatermarkForEviction = None,
+          isRealTimeMode = IncrementalExecution.this.isRealTimeMode,
           hasInitialState = hasInitialState
         )
 
@@ -692,6 +719,7 @@ class IncrementalExecution(
         checkOperatorValidWithMetadata(planWithStateOpId, currentBatchId - 1)
       }
 
+      stateStoreWritersWithMetadata = Some(Seq.empty)
       val planWithSchemas = planWithStateOpId transform StateSchemaAndOperatorMetadataRule.rule
 
       simulateWatermarkPropagation(planWithSchemas)
@@ -699,6 +727,32 @@ class IncrementalExecution(
     }
   }
 
+  private def isTransformWithStateInitialStateBootstrap: Boolean = {
+    isRealTimeMode && currentBatchId == 0 && logicalPlan.exists {
+      case tws: TransformWithState => tws.hasInitialState
+      case _ => false
+    }
+  }
+
+  /**
+   * The initial state is loaded in a finite batch before the Real-Time Mode source starts its
+   * first long-running batch. This lets the initial-state shuffle materialize and prevents input
+   * that was already available when the query started from being processed before initialization.
+   * The pipelined-shuffle rule also skips this batch because the DAGScheduler does not support a
+   * job that mixes the initial state's regular shuffle with a pipelined streaming shuffle.
+   */
+  object PrepareTransformWithStateInitialStateForRealTimeMode extends Rule[SparkPlan] {
+    override def apply(plan: SparkPlan): SparkPlan = {
+      if (isTransformWithStateInitialStateBootstrap) {
+        plan.transformUp {
+          case scan: RealTimeStreamScanExec => scan.copy(batchDurationMs = 0L)
+        }
+      } else {
+        plan
+      }
+    }
+  }
+
   /**
    * For a Real-Time Mode batch, mark the shuffle exchanges as pipelined so the DAGScheduler
    * co-schedules a stateful query's producer (source scan) and consumer (stateful operator) stages
@@ -744,7 +798,7 @@ class IncrementalExecution(
   object MarkPipelinedShuffleForRealTimeMode extends Rule[SparkPlan] {
     override def apply(plan: SparkPlan): SparkPlan = {
       val isRealTimeMode = plan.exists(_.isInstanceOf[RealTimeStreamScanExec])
-      if (!isRealTimeMode) {
+      if (!isRealTimeMode || isTransformWithStateInitialStateBootstrap) {
         plan
       } else {
         markStreamingPath(plan)._1
@@ -787,7 +841,9 @@ class IncrementalExecution(
   }
 
   override def preparations: Seq[Rule[SparkPlan]] =
-    state +: (super.preparations :+ MarkPipelinedShuffleForRealTimeMode)
+    state +: (super.preparations :+
+      PrepareTransformWithStateInitialStateForRealTimeMode :+
+      MarkPipelinedShuffleForRealTimeMode)
 
   /** no need to try-catch again as this is already done once */
   override def assertAnalyzed(): Unit = analyzed
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/MicroBatchExecution.scala
index f0201ccf302ac..e7500e6d634d6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/MicroBatchExecution.scala
@@ -565,20 +565,20 @@ class MicroBatchExecution(
       CommitLogType,
       commitLogFormatVersion,
       latestCommittedBatch.map(_._2))
+    val stateStoreCheckpointFormatVersion =
+      sparkSessionForStream.sessionState.conf.stateStoreCheckpointFormatVersion
 
-    // Real-Time Mode requires commit log v2. Real-Time Mode writes the offset log at batch end
+    // Real-Time Mode requires state store checkpoint format v2. It writes the offset log at
+    // batch end
     // (markMicroBatchStart is a no-op for it), so a mid-batch failure can leave durable state at a
     // version that was never logged; the re-execution then rewrites that same state version. With
     // checkpoint format v1 the rewritten files reuse the same names as the orphaned ones, so a load
     // can pick up a stale file (see the checksum hazard documented on
     // StateStoreConf.skipChecksumOnFileMissingChecksum). Format v2 avoids this because each batch
-    // run generates unique state store checkpoint ids, and only a commit log at v2 or above can
-    // persist them. Resolution above keeps an existing checkpoint at the version it was created
-    // with, so a v1 checkpoint stays v1. Reject any Real-Time Mode query whose resolved commit log
-    // version is below v2, with an escape hatch. This is unconditional, matching the Databricks
-    // runtime -- a fresh checkpoint reaches v1 here only when the user explicitly pinned it, which
-    // is exactly the case worth rejecting.
-    if (trigger.isInstanceOf[RealTimeTrigger] && commitLogFormatVersion < CommitLog.VERSION_2) {
+    // run generates unique state store checkpoint ids. Commit log v2 persists those ids; a v3
+    // commit may or may not contain them, so the resolved state store format is the authoritative
+    // check. Reject v1 with an escape hatch.
+    if (trigger.isInstanceOf[RealTimeTrigger] && stateStoreCheckpointFormatVersion < 2) {
       if (!sparkSessionForStream.sessionState.conf
           .getConf(SQLConf.STREAMING_REAL_TIME_MODE_DANGEROUSLY_ALLOW_CHECKPOINT_V1)) {
         throw new SparkIllegalArgumentException(
@@ -586,8 +586,8 @@ class MicroBatchExecution(
           messageParameters = Map(
             "config" -> SQLConf.STREAMING_REAL_TIME_MODE_DANGEROUSLY_ALLOW_CHECKPOINT_V1.key))
       }
-      logWarning(log"Starting a Real-Time Mode query on a commit log at version " +
-        log"${MDC(LogKeys.FILE_VERSION, commitLogFormatVersion)} because " +
+      logWarning(log"Starting a Real-Time Mode query on state store checkpoint format version " +
+        log"${MDC(LogKeys.FILE_VERSION, stateStoreCheckpointFormatVersion)} because " +
         log"${MDC(LogKeys.CONFIG, SQLConf.STREAMING_REAL_TIME_MODE_DANGEROUSLY_ALLOW_CHECKPOINT_V1
           .key)} is set. A failed batch may lose data on rerun.")
     }
@@ -907,8 +907,13 @@ class MicroBatchExecution(
   private def verifyNewCheckpointDirectory(): Unit = {
     val fileManager = CheckpointFileManager.create(new Path(resolvedCheckpointRoot),
       sparkSession.sessionState.newHadoopConf())
-    val dirNamesThatShouldNotHaveFiles = Array[String](
-      DIR_NAME_OFFSETS, DIR_NAME_STATE, DIR_NAME_COMMITS)
+    var dirNamesThatShouldNotHaveFiles = Array[String](DIR_NAME_OFFSETS, DIR_NAME_COMMITS)
+
+    // Since real-time mode writes the offset log after the batch is committed, the state directory
+    // may contain files so we want to allow the streaming query to retry.
+    if (!trigger.isInstanceOf[RealTimeTrigger]) {
+      dirNamesThatShouldNotHaveFiles :+= DIR_NAME_STATE
+    }
 
     dirNamesThatShouldNotHaveFiles.foreach { dirName =>
       val path = new Path(resolvedCheckpointRoot, dirName)
@@ -1277,7 +1282,8 @@ class MicroBatchExecution(
         execCtx.previousContext.isEmpty,
         currentStateStoreCkptId,
         stateSchemaMetadatas,
-        isTerminatingTrigger = trigger.isInstanceOf[AvailableNowTrigger.type])
+        isTerminatingTrigger = trigger.isInstanceOf[AvailableNowTrigger.type],
+        isRealTimeMode = trigger.isInstanceOf[RealTimeTrigger])
       execCtx.executionPlan.executedPlan // Force the lazy generation of execution plan
     }
     // Set up StateStore commit tracking before execution begins
@@ -1502,6 +1508,11 @@ class MicroBatchExecution(
         log"Committed offsets for batch ${MDC(LogKeys.BATCH_ID, execCtx.batchId)}. Metadata " +
         log"${MDC(LogKeys.OFFSET_SEQUENCE_METADATA, execCtx.offsetSeqMetadata)}"
       )
+
+      // State schema validation and broadcast creation still happen during planning, but RTM
+      // defers operator metadata until its end offset is durable. This prevents a failed batch
+      // from leaving metadata that has no corresponding offset log entry.
+      execCtx.executionPlan.writeRecordedStateMetadata()
     }
 
     execCtx.reportTimeTaken("commitOffsets") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/RealTimeModeAllowlist.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/RealTimeModeAllowlist.scala
index 47cc286bb6170..8b7853b7f03d6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/RealTimeModeAllowlist.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/runtime/RealTimeModeAllowlist.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.v2.RealTimeStreamScanExec
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.execution.streaming.operators.stateful._
+import org.apache.spark.sql.execution.streaming.operators.stateful.transformwithstate.TransformWithStateExec
 
 object RealTimeModeAllowlist extends Logging {
   private val allowedSinks = Set(
@@ -75,7 +76,9 @@ object RealTimeModeAllowlist extends Logging {
     "org.apache.spark.sql.execution.streaming.operators.stateful.StateStoreRestoreExec",
     "org.apache.spark.sql.execution.streaming.operators.stateful.StateStoreSaveExec",
     "org.apache.spark.sql.execution.streaming.operators.stateful.StreamingDeduplicateExec",
-    classOf[EventTimeWatermarkExec].getName
+    classOf[EventTimeWatermarkExec].getName,
+    classOf[TransformWithStateExec].getName,
+    classOf[UpdateEventTimeColumnExec].getName
   )
 
   private def classNamesString(classNames: Seq[String]): MessageWithContext = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala
index f3c7fdda1a1ce..eeb8ce298c41f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDB.scala
@@ -1658,6 +1658,36 @@ class RocksDB(
     }
   }
 
+  /**
+   * Return an iterator that remains open when exhausted so it can be refreshed and reused.
+   */
+  private[state] def reusableIterator(
+      cfName: String = StateStore.DEFAULT_COL_FAMILY_NAME): RocksDBIterator = {
+    updateMemoryUsageIfNeeded()
+    val virtualColumnFamilyId = if (useColumnFamilies) {
+      Some(getColumnFamilyInfo(cfName).cfId)
+    } else {
+      None
+    }
+    val reusableIterator = new RocksDBIterator(
+      db.newIterator(),
+      useColumnFamilies,
+      virtualColumnFamilyId,
+      conf.rowChecksumEnabled,
+      readVerifier,
+      delimiterSize)
+    if (useColumnFamilies) {
+      reusableIterator.seek(Array.emptyByteArray)
+    } else {
+      reusableIterator.seekToFirst()
+    }
+
+    Option(TaskContext.get()).foreach { tc =>
+      tc.addTaskCompletionListener[Unit] { _ => reusableIterator.close() }
+    }
+    reusableIterator
+  }
+
   private def countKeys(): (Long, Long) = {
     val iter = db.newIterator()
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBIterator.scala
new file mode 100644
index 0000000000000..f0fcd45b25589
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBIterator.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.streaming.state
+
+import org.rocksdb.{RocksIterator => NativeRocksIterator}
+
+/**
+ * A RocksDB iterator that can be refreshed and repositioned without recreating its native
+ * resources.
+ */
+private[state] class RocksDBIterator(
+    iter: NativeRocksIterator,
+    useColumnFamilies: Boolean,
+    virtualColumnFamilyId: Option[Short],
+    rowChecksumEnabled: Boolean,
+    readVerifier: Option[KeyValueIntegrityVerifier],
+    delimiterSize: Int) extends Iterator[ByteArrayPair] with AutoCloseable {
+
+  private val byteArrayPair = new ByteArrayPair()
+
+  def refresh(): Unit = iter.refresh()
+
+  def seek(key: Array[Byte]): Unit = {
+    val encodedKey = virtualColumnFamilyId match {
+      case Some(id) => RocksDBStateStoreProvider.encodeStateRowWithPrefix(key, id)
+      case None => key
+    }
+    iter.seek(encodedKey)
+  }
+
+  def seekToFirst(): Unit = iter.seekToFirst()
+
+  override def hasNext: Boolean = {
+    iter.isValid && virtualColumnFamilyId.forall { id =>
+      RocksDBStateStoreProvider.getColumnFamilyBytesAsId(iter.key()) == id
+    }
+  }
+
+  override def next(): ByteArrayPair = {
+    val key = if (useColumnFamilies) {
+      RocksDBStateStoreProvider.decodeStateRowWithPrefix(iter.key())
+    } else {
+      iter.key()
+    }
+    val value = if (rowChecksumEnabled) {
+      KeyValueChecksumEncoder.decodeAndVerifyValueRowWithChecksum(
+        readVerifier, iter.key(), iter.value(), delimiterSize)
+    } else {
+      iter.value()
+    }
+
+    byteArrayPair.set(key, value)
+    iter.next()
+    byteArrayPair
+  }
+
+  override def close(): Unit = iter.close()
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala
index 36f64d1175dc0..79c69634509bb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala
@@ -47,7 +47,8 @@ private[sql] class RocksDBStateStoreProvider
       lastVersion: Long,
       private[RocksDBStateStoreProvider] val stamp: Long,
       private[RocksDBStateStoreProvider] var readOnly: Boolean,
-      private[RocksDBStateStoreProvider] var forceSnapshotOnCommit: Boolean) extends StateStore {
+      private[RocksDBStateStoreProvider] var forceSnapshotOnCommit: Boolean)
+    extends StateStore with SupportsReusableIterator {
 
     private sealed trait OPERATION
     private case object UPDATE extends OPERATION
@@ -490,6 +491,49 @@ private[sql] class RocksDBStateStoreProvider
       }
     }
 
+    private def wrapReusableIterator(
+        rocksDbIter: RocksDBIterator,
+        keyEncoder: RocksDBKeyStateEncoder): ReusableIterator[ByteArrayPair] = {
+      new ReusableIterator[ByteArrayPair] {
+        override def refreshAndSeekToPrefix(prefixRow: UnsafeRow): Unit = {
+          rocksDbIter.refresh()
+          val encoded = keyEncoder match {
+            case rangeKeyScanStateEncoder: RangeKeyScanStateEncoder =>
+              rangeKeyScanStateEncoder.encodePrefixKey(prefixRow)
+            case _ =>
+              throw new StateStoreUnsupportedOperationException(
+                "refreshAndSeekToPrefix", keyEncoder.getClass.getName)
+          }
+          rocksDbIter.seek(encoded)
+        }
+
+        override def hasNext: Boolean = rocksDbIter.hasNext
+
+        override def next(): ByteArrayPair = rocksDbIter.next()
+
+        override def close(): Unit = rocksDbIter.close()
+      }
+    }
+
+    override def reusableIterator(colFamilyName: String): ReusableIterator[UnsafeRowPair] = {
+      validateAndTransitionState(UPDATE)
+      verifyColFamilyOperations("iterator", colFamilyName)
+
+      val kvEncoder = keyValueEncoderMap.get(colFamilyName)
+      val rowPair = new UnsafeRowPair()
+      wrapReusableIterator(rocksDB.reusableIterator(colFamilyName), kvEncoder._1).map { kv =>
+        rowPair.withRows(
+          kvEncoder._1.decodeKey(kv.key),
+          kvEncoder._2.decodeValue(kv.value))
+        if (!isValidated && rowPair.value != null && !useColumnFamilies) {
+          StateStoreProvider.validateStateRowFormat(
+            rowPair.key, keySchema, rowPair.value, valueSchema, stateStoreId, storeConf)
+          isValidated = true
+        }
+        rowPair
+      }
+    }
+
     override def iteratorWithMultiValues(
         colFamilyName: String): StateStoreIterator[UnsafeRowPair] = {
       validateAndTransitionState(UPDATE)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala
index 582e1a1038a0c..9c969efe93191 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala
@@ -68,6 +68,34 @@ class StateStoreIterator[A](
   override def close(): Unit = onClose()
 }
 
+/**
+ * An iterator that can be refreshed and repositioned instead of recreated for every scan.
+ */
+private[sql] abstract class ReusableIterator[A] extends Iterator[A] with Closeable {
+  /** Refresh this iterator and seek to the encoded prefix row. */
+  def refreshAndSeekToPrefix(prefixRow: UnsafeRow): Unit
+
+  override def map[B](f: A => B): ReusableIterator[B] = {
+    val self = this
+    new ReusableIterator[B] {
+      override def refreshAndSeekToPrefix(prefixRow: UnsafeRow): Unit =
+        self.refreshAndSeekToPrefix(prefixRow)
+
+      override def hasNext: Boolean = self.hasNext
+
+      override def next(): B = f(self.next())
+
+      override def close(): Unit = self.close()
+    }
+  }
+}
+
+/** State stores that can return an iterator whose native resources are reused across scans. */
+private[sql] trait SupportsReusableIterator {
+  def reusableIterator(
+      colFamilyName: String = StateStore.DEFAULT_COL_FAMILY_NAME): ReusableIterator[UnsafeRowPair]
+}
+
 sealed trait StateStoreEncoding {
   override def toString: String = this match {
     case StateStoreEncoding.UnsafeRow => "unsaferow"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/state/StateDataSourceRealTimeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/state/StateDataSourceRealTimeSuite.scala
new file mode 100644
index 0000000000000..71714e1eb550c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/state/StateDataSourceRealTimeSuite.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.v2.state
+
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.streaming.sources.{ContinuousMemorySink,
+  LowLatencyMemoryStream}
+import org.apache.spark.sql.execution.streaming.state.RocksDBStateStoreProvider
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.streaming.{OutputMode, StreamRealTimeModeManualClockSuiteBase, TimeMode}
+
+/**
+ * State data source reads under Real-Time Mode (RTM).
+ */
+class StateDataSourceRealTimeSuite
+    extends StreamRealTimeModeManualClockSuiteBase
+    with StateDataSourceTestBase {
+
+  import testImplicits._
+
+  private def assertEventuallyBatchCommitted(batchId: Long): StreamAction = {
+    Execute(s"Assert batch $batchId is committed") { q =>
+      eventually(timeout(1.minute)) {
+        assert(q.commitLog.getLatest().get._1 === batchId)
+      }
+    }
+  }
+
+  test("transformWithState + RTM: state data source read") {
+    withSQLConf(
+        SQLConf.STATE_STORE_PROVIDER_CLASS.key ->
+          classOf[RocksDBStateStoreProvider].getName,
+        SQLConf.SHUFFLE_PARTITIONS.key -> "2",
+        SQLConf.STATE_STORE_CHECKPOINT_FORMAT_VERSION.key -> "2") {
+      withTempDir { tempDir =>
+        val input = LowLatencyMemoryStream[String](2)
+        val query = input.toDS()
+          .groupByKey(x => x)
+          .transformWithState(
+            new StatefulProcessorWithSingleValueVar(),
+            TimeMode.ProcessingTime(),
+            OutputMode.Update())
+        val checkpoint = tempDir.getCanonicalPath
+
+        testStream(query, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(trigger = defaultTrigger, checkpointLocation = checkpoint),
+          AddData(input, "a", "b"),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "1"), ("b", "1")),
+          advanceRealTimeClock,
+          assertEventuallyBatchCommitted(0),
+          StopStream
+        )
+
+        val stateReaderDf = spark.read
+          .format("statestore")
+          .option(StateSourceOptions.PATH, checkpoint)
+          .option(StateSourceOptions.STATE_VAR_NAME, "valueState")
+          .load()
+
+        checkAnswer(
+          stateReaderDf.selectExpr(
+            "key.value AS groupingKey",
+            "value.id AS valueId",
+            "value.name AS valueName"),
+          Seq(Row("a", 1L, "dummyKey"), Row("b", 1L, "dummyKey")))
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/TimerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/TimerSuite.scala
index 8475f283b6628..86e39c668df83 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/TimerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/TimerSuite.scala
@@ -60,6 +60,33 @@ class TimerSuite extends StateVariableSuiteBase {
     }
   }
 
+  testWithTimeMode("reusable expired timer iterator resumes from prior threshold") { timeMode =>
+    tryWithProviderResource(newStoreProviderWithStateVariable(true)) { provider =>
+      val store = provider.getStore(0)
+      assert(store.isInstanceOf[SupportsReusableIterator])
+
+      ImplicitGroupingKeyTracker.setImplicitKey("test_key")
+      val timerState = new TimerStateImpl(store, timeMode, stringEncoder)
+      timerState.registerTimer(1000L)
+      timerState.registerTimer(3000L)
+
+      assert(timerState.getExpiredTimersReusable(1500L).toSeq ===
+        Seq(("test_key", 1000L)))
+
+      timerState.deleteTimer(1000L)
+      // The refreshed iterator resumes at the prior 1500 ms threshold, so it does not revisit
+      // the backdated timer at 500 ms.
+      timerState.registerTimer(500L)
+      timerState.registerTimer(1500L)
+      timerState.registerTimer(2000L)
+
+      assert(timerState.getExpiredTimersReusable(2500L).toSeq ===
+        Seq(("test_key", 1500L), ("test_key", 2000L)))
+      assert(timerState.getExpiredTimers(2500L).toSeq ===
+        Seq(("test_key", 500L), ("test_key", 1500L), ("test_key", 2000L)))
+    }
+  }
+
   testWithTimeMode("multiple instances with single key") { timeMode =>
     tryWithProviderResource(newStoreProviderWithStateVariable(true)) { provider =>
       val store = provider.getStore(0)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/ValueStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/ValueStateSuite.scala
index 874c69dee1d61..77cad5ea5ab67 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/ValueStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/ValueStateSuite.scala
@@ -371,6 +371,43 @@ class ValueStateSuite extends StateVariableSuiteBase {
     }
   }
 
+  test("Value state TTL uses the current processing-time function") {
+    tryWithProviderResource(newStoreProviderWithStateVariable(true)) { provider =>
+      val store = provider.getStore(0)
+      var currentTimestampMs = 1000L
+      val handle = new StatefulProcessorHandleImpl(
+        store,
+        UUID.randomUUID(),
+        stringEncoder,
+        TimeMode.ProcessingTime(),
+        batchTimestampMs = Some(currentTimestampMs),
+        currentTimestampMs = Some(() => currentTimestampMs))
+      val state = handle.getValueState[String](
+        "testState", Encoders.STRING, TTLConfig(Duration.ofMillis(100)))
+        .asInstanceOf[ValueStateImplWithTTL[String]]
+
+      ImplicitGroupingKeyTracker.setImplicitKey("test_key")
+      try {
+        state.update("v1")
+        assert(state.getTTLValue().contains(("v1", 1100L)))
+
+        currentTimestampMs = 1099L
+        assert(state.get() === "v1")
+        currentTimestampMs = 1100L
+        assert(state.get() === null)
+
+        state.update("v2")
+        assert(state.getTTLValue().contains(("v2", 1200L)))
+        currentTimestampMs = 1199L
+        assert(state.get() === "v2")
+        currentTimestampMs = 1200L
+        assert(state.get() === null)
+      } finally {
+        ImplicitGroupingKeyTracker.removeImplicitKey()
+      }
+    }
+  }
+
   // Guards against the UnsafeRow byte-order bug where a scan boundary row with a
   // null element-key struct encodes larger than a real entry (null-bitmap bit = 1),
   // making seek() silently skip boundary entries. Uses a primitive Long grouping
@@ -411,7 +448,7 @@ class ValueStateSuite extends StateVariableSuiteBase {
         Seq(firstBatchTs + 1, firstBatchTs + 1, firstBatchTs + 1))
 
       // The eviction iterator (bounded range scan) should find all three entries.
-      val evicted = state2.ttlEvictionIterator().toList
+      val evicted = state2.ttlEvictionIterator(nextBatchTs).toList
       assert(evicted.size === 3,
         s"Expected 3 evictable TTL entries at expiration = prevBatch + 1, got ${evicted.size}")
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/RealTimeTransformWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/RealTimeTransformWithStateSuite.scala
new file mode 100644
index 0000000000000..b603d47ecf154
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/RealTimeTransformWithStateSuite.scala
@@ -0,0 +1,2050 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.sql.Timestamp
+import java.time.Duration
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import org.apache.hadoop.fs.Path
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.{
+  SparkConf, SparkException, SparkRuntimeException, SparkThrowable, TaskContext, TaskContextImpl}
+import org.apache.spark.sql.Encoders
+import org.apache.spark.sql.execution.SortExec
+import org.apache.spark.sql.execution.datasources.v2.{LowLatencyClock, RealTimeStreamScanExec}
+import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.streaming.operators.stateful.transformwithstate.TransformWithStateExec
+import org.apache.spark.sql.execution.streaming.sources.{ContinuousMemorySink, LowLatencyMemoryStream}
+import org.apache.spark.sql.execution.streaming.state.{
+  EnableStateStoreRowChecksum, OperatorStateMetadataReader, OperatorStateMetadataV2,
+  OperatorStateMetadataWriter, RocksDBConf, RocksDBStateStoreProvider}
+import org.apache.spark.sql.functions.col
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.streaming.util.{GlobalSingletonManualClock, StreamManualClock}
+
+private class RealTimeEagerCountProcessor
+  extends StatefulProcessor[String, (String, Int), (String, Long)] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState("count", Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, Long)] = {
+    // Eager consumption detects the blocking final group produced by GroupedIterator.
+    val newCount = Option(countState.get()).getOrElse(0L) + inputRows.size
+    countState.update(newCount)
+    // Access state lazily as the output is consumed to verify implicit-key lifecycle handling.
+    Iterator.single(key).map(currentKey => (currentKey, countState.get()))
+  }
+}
+
+private class RealTimeRunningCountStatefulProcessor(emitEvery: Long)
+  extends StatefulProcessor[String, String, (String, Long)] {
+
+  @transient private var countState: MapState[String, Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getMapState(
+      "countState", Encoders.STRING, Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[String],
+      timerValues: TimerValues): Iterator[(String, Long)] = {
+    inputRows.flatMap { row =>
+      val count = countState.getValue(row) + 1L
+      countState.updateValue(row, count)
+      if (count % emitEvery == 0L) Iterator.single((row, count)) else Iterator.empty
+    }
+  }
+}
+
+private class RealTimeTTLCountProcessor(ttl: Duration = Duration.ofSeconds(10))
+  extends StatefulProcessor[String, (String, Int), (String, Long)] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState(
+      "count", Encoders.scalaLong, TTLConfig(ttl))
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, Long)] = {
+    inputRows.map { _ =>
+      val newCount = Option(countState.get()).getOrElse(0L) + 1L
+      countState.update(newCount)
+      (key, newCount)
+    }
+  }
+}
+
+private class RealTimeListTTLProcessor
+  extends StatefulProcessor[String, (String, Int), (String, Long)] {
+
+  @transient private var listState: ListState[Int] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    listState = getHandle.getListState(
+      "values", Encoders.scalaInt, TTLConfig(Duration.ofSeconds(10)))
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, Long)] = {
+    inputRows.map { case (_, value) =>
+      listState.appendList(Array(value, value + 1, value + 2))
+      (key, listState.get().size.toLong)
+    }
+  }
+}
+
+private class RealTimeMapTTLAndTimerProcessor
+  extends StatefulProcessor[String, (String, Int), (String, String, Long)] {
+
+  @transient private var countState: MapState[String, Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getMapState(
+      "count", Encoders.STRING, Encoders.scalaLong, TTLConfig(Duration.ofMinutes(10)))
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, String, Long)] = {
+    inputRows.map { case (_, timerDelayMs) =>
+      val count = Option(countState.getValue("count")).getOrElse(0L) + 1L
+      countState.updateValue("count", count)
+      getHandle.registerTimer(timerValues.getCurrentProcessingTimeInMs() + timerDelayMs)
+      (key, "data", count)
+    }
+  }
+
+  override def handleExpiredTimer(
+      key: String,
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(String, String, Long)] = {
+    Iterator.single((key, "timer", expiredTimerInfo.getExpiryTimeInMs()))
+  }
+}
+
+private class RealTimePartitionProcessor
+  extends StatefulProcessor[String, (String, Int), (String, Int)] {
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {}
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, Int)] = {
+    inputRows.map(_ => (key, TaskContext.getPartitionId()))
+  }
+}
+
+private class RealTimeProcessingTimerProcessor
+  extends StatefulProcessor[String, (String, Int), (String, String)] {
+
+  @transient private var timerRegistered: ValueState[Boolean] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    timerRegistered =
+      getHandle.getValueState("timerRegistered", Encoders.scalaBoolean, TTLConfig.NONE)
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, String)] = {
+    inputRows.map { _ =>
+      if (!Option(timerRegistered.get()).getOrElse(false)) {
+        getHandle.registerTimer(timerValues.getCurrentProcessingTimeInMs() + 10000L)
+        timerRegistered.update(true)
+      }
+      (key, "data")
+    }
+  }
+
+  override def handleExpiredTimer(
+      key: String,
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(String, String)] = {
+    Iterator.single((key, "timer"))
+  }
+}
+
+private class RealTimeProcessingTimerValueProcessor
+  extends StatefulProcessor[String, (String, Int), (String, String, Long)] {
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {}
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, String, Long)] = {
+    inputRows.map { _ =>
+      val currentTimeMs = timerValues.getCurrentProcessingTimeInMs()
+      getHandle.registerTimer(currentTimeMs + 10000L)
+      (key, "data", currentTimeMs)
+    }
+  }
+
+  override def handleExpiredTimer(
+      key: String,
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(String, String, Long)] = {
+    Iterator.single((key, "timer", timerValues.getCurrentProcessingTimeInMs()))
+  }
+}
+
+private object TaskCompletionListenerCount {
+  private lazy val listenerStackField = {
+    val field = classOf[TaskContextImpl].getDeclaredField("onCompleteCallbacks")
+    field.setAccessible(true)
+    field
+  }
+
+  def get(): Int = listenerStackField.get(TaskContext.get())
+    .asInstanceOf[java.util.Stack[_]].size()
+}
+
+private class RealTimeTimerIteratorListenerProcessor
+  extends StatefulProcessor[String, (String, Int), (Int, Boolean)] {
+
+  private var previousListenerCount = -1
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {}
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(Int, Boolean)] = {
+    inputRows.map { case (_, value) =>
+      val listenerCount = TaskCompletionListenerCount.get()
+      val listenerCountIncreased =
+        previousListenerCount >= 0 && listenerCount > previousListenerCount
+      previousListenerCount = listenerCount
+      (value, listenerCountIncreased)
+    }
+  }
+}
+
+private class RTMStatefulProcessorWithProcTimeTimerWithMultipleTimers(timerExpireTs: Long)
+    extends RTMStatefulProcessorWithProcTimeTimer(timerExpireTs) {
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[String],
+      timerValues: TimerValues): Iterator[(String, String)] = {
+
+    val currCount = Option(_countState.get()).getOrElse(0L)
+    if (currCount == 0 && (key == "a" || key == "c")) {
+      getHandle.registerTimer(
+        timerValues.getCurrentProcessingTimeInMs() + timerExpireTs
+      )
+
+      getHandle.registerTimer(
+        timerValues.getCurrentProcessingTimeInMs() + (timerExpireTs + 1000)
+      )
+    }
+
+    val count = currCount + 1
+    if (count == 3) {
+      _countState.clear()
+      Iterator.empty
+    } else {
+      _countState.update(count)
+      Iterator((key, count.toString))
+    }
+  }
+}
+
+private class RTMStatefulProcessorWithProcTimeTimer(timerExpireTs: Long)
+    extends RunningCountStatefulProcessor {
+
+  override def handleExpiredTimer(
+      key: String,
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(String, String)] = {
+    _countState.clear()
+    Iterator((key, "-1"))
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[String],
+      timerValues: TimerValues): Iterator[(String, String)] = {
+
+    val currCount = Option(_countState.get()).getOrElse(0L)
+    if (currCount == 0 && (key == "a" || key == "c")) {
+      getHandle.registerTimer(
+        timerValues.getCurrentProcessingTimeInMs() + timerExpireTs
+      )
+    }
+
+    val count = currCount + 1
+    if (count == 3) {
+      _countState.clear()
+      Iterator.empty
+    } else {
+      _countState.update(count)
+      Iterator((key, count.toString))
+    }
+  }
+}
+
+private class RTMStatefulProcessorWithProcTimeTimerInputInt(timerExpireTs: Long)
+    extends StatefulProcessor[Int, Int, (Int, Long)] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState(
+      "countState", Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleExpiredTimer(
+      key: Int,
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(Int, Long)] = {
+    countState.clear()
+    Iterator.single((key, -1L))
+  }
+
+  override def handleInputRows(
+      key: Int,
+      inputRows: Iterator[Int],
+      timerValues: TimerValues): Iterator[(Int, Long)] = {
+    val currentCount = Option(countState.get()).getOrElse(0L)
+    if (currentCount == 0L) {
+      getHandle.registerTimer(timerValues.getCurrentProcessingTimeInMs() + timerExpireTs)
+    }
+    val count = currentCount + 1L
+    if (count == 3L) {
+      countState.clear()
+      Iterator.empty
+    } else {
+      countState.update(count)
+      Iterator.single((key, count))
+    }
+  }
+}
+
+private class RTMStatefulProcessorWithProcTimeTimerInputTuple(timerExpireTs: Long)
+    extends StatefulProcessor[(Int, String), (Int, String), (Int, String, Long)] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState(
+      "countState", Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleExpiredTimer(
+      key: (Int, String),
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(Int, String, Long)] = {
+    countState.clear()
+    Iterator.single((key._1, key._2, -1L))
+  }
+
+  override def handleInputRows(
+      key: (Int, String),
+      inputRows: Iterator[(Int, String)],
+      timerValues: TimerValues): Iterator[(Int, String, Long)] = {
+    val currentCount = Option(countState.get()).getOrElse(0L)
+    if (currentCount == 0L) {
+      getHandle.registerTimer(timerValues.getCurrentProcessingTimeInMs() + timerExpireTs)
+    }
+    val count = currentCount + 1L
+    if (count == 3L) {
+      countState.clear()
+      Iterator.empty
+    } else {
+      countState.update(count)
+      Iterator.single((key._1, key._2, count))
+    }
+  }
+}
+
+private class RealTimeEventTimerProcessor
+  extends StatefulProcessor[String, (Timestamp, String), (String, String, Long)] {
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {}
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(Timestamp, String)],
+      timerValues: TimerValues): Iterator[(String, String, Long)] = {
+    inputRows.map { case (eventTime, _) =>
+      getHandle.registerTimer(eventTime.getTime)
+      (key, "data", timerValues.getCurrentWatermarkInMs())
+    }
+  }
+
+  override def handleExpiredTimer(
+      key: String,
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(String, String, Long)] = {
+    Iterator.single((key, "timer", timerValues.getCurrentWatermarkInMs()))
+  }
+}
+
+private case class RealTimeEventTimeOutputRow(
+    key: String,
+    outputEventTime: Timestamp,
+    count: Long)
+
+private class RealTimeEventTimeOutputProcessor(
+    outputEventTimeOverride: Option[Timestamp] = None)
+  extends StatefulProcessor[String, (Timestamp, String), RealTimeEventTimeOutputRow] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState("count", Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(Timestamp, String)],
+      timerValues: TimerValues): Iterator[RealTimeEventTimeOutputRow] = {
+    inputRows.map { case (eventTime, _) =>
+      val newCount = Option(countState.get()).getOrElse(0L) + 1L
+      countState.update(newCount)
+      RealTimeEventTimeOutputRow(
+        key, outputEventTimeOverride.getOrElse(eventTime), newCount)
+    }
+  }
+}
+
+private class RealTimeInitialCountProcessor
+  extends StatefulProcessorWithInitialState[
+    String, (String, Int), (String, Long), Long] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState("count", Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleInitialState(
+      key: String,
+      initialState: Long,
+      timerValues: TimerValues): Unit = {
+    countState.update(Option(countState.get()).getOrElse(0L) + initialState)
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(String, Int)],
+      timerValues: TimerValues): Iterator[(String, Long)] = {
+    inputRows.map { _ =>
+      val newCount = Option(countState.get()).getOrElse(0L) + 1L
+      countState.update(newCount)
+      (key, newCount)
+    }
+  }
+}
+
+private class RealTimeEventInitialCountProcessor
+  extends StatefulProcessorWithInitialState[
+    String, (Timestamp, String), (String, Long), Long] {
+
+  @transient private var countState: ValueState[Long] = _
+
+  override def init(outputMode: OutputMode, timeMode: TimeMode): Unit = {
+    countState = getHandle.getValueState("count", Encoders.scalaLong, TTLConfig.NONE)
+  }
+
+  override def handleInitialState(
+      key: String,
+      initialState: Long,
+      timerValues: TimerValues): Unit = {
+    countState.update(initialState)
+  }
+
+  override def handleInputRows(
+      key: String,
+      inputRows: Iterator[(Timestamp, String)],
+      timerValues: TimerValues): Iterator[(String, Long)] = {
+    inputRows.map { _ =>
+      val newCount = Option(countState.get()).getOrElse(0L) + 1L
+      countState.update(newCount)
+      (key, newCount)
+    }
+  }
+}
+
+private class RealTimeInitialStateProcTimerWithExpiryProcessor
+  extends StatefulProcessorWithInitialStateProcTimerClass {
+
+  override def handleExpiredTimer(
+      key: String,
+      timerValues: TimerValues,
+      expiredTimerInfo: ExpiredTimerInfo): Iterator[(String, String)] = {
+    super.handleExpiredTimer(key, timerValues, expiredTimerInfo).map { case (expiredKey, _) =>
+      (expiredKey, expiredTimerInfo.getExpiryTimeInMs().toString)
+    }
+  }
+}
+
+private object RealTimeInitialStateFailure {
+  @volatile var enabled: Boolean = false
+}
+
+private object RealTimeInitialStateBootstrapBlock {
+  @volatile private var enabled = false
+  @volatile private var taskStarted = new CountDownLatch(0)
+  @volatile private var releaseTask = new CountDownLatch(0)
+
+  def enable(): Unit = {
+    taskStarted = new CountDownLatch(1)
+    releaseTask = new CountDownLatch(1)
+    enabled = true
+  }
+
+  def awaitTaskStart(): Boolean = taskStarted.await(1, TimeUnit.MINUTES)
+
+  def awaitReleaseIfEnabled(): Unit = {
+    if (enabled) {
+      taskStarted.countDown()
+      releaseTask.await()
+    }
+  }
+
+  def disable(): Unit = {
+    enabled = false
+    releaseTask.countDown()
+  }
+}
+
+class RealTimeTransformWithStateSuite extends StreamRealTimeModeE2ESuiteBase {
+  import testImplicits._
+
+  override protected def sparkConf: SparkConf = super.sparkConf
+    .set(SQLConf.STATE_STORE_PROVIDER_CLASS.key, classOf[RocksDBStateStoreProvider].getName)
+
+  private def advanceClock(clock: GlobalSingletonManualClock): ExternalAction = {
+    advanceClock(clock, defaultTrigger.batchDurationMs)
+  }
+
+  private def advanceClock(
+      clock: GlobalSingletonManualClock,
+      advanceMs: Long): ExternalAction = {
+    new ExternalAction {
+      override def runAction(): Unit = clock.advance(advanceMs)
+    }
+  }
+
+  private def createStringMemoryStream(numPartitions: Int = 2)
+      : (LowLatencyMemoryStream[String], GlobalSingletonManualClock) = {
+    val clock = new GlobalSingletonManualClock()
+    LowLatencyClock.setClock(clock)
+    (LowLatencyMemoryStream[String](numPartitions), clock)
+  }
+
+  private def waitForNextBatchToStart(): Unit = {
+    eventually(timeout(60.seconds)) {
+      val tasksRunning = spark.sparkContext.statusTracker
+        .getExecutorInfos.map(_.numRunningTasks()).sum
+      assert(tasksRunning >= 1, s"tasksRunning: $tasksRunning")
+    }
+  }
+
+  private def initialState(values: Seq[(String, Long)]) = {
+    values.toDS()
+      // More initial-state partitions than available task slots exercises finite bootstrap
+      // scheduling independently from the later pipelined RTM batches.
+      .repartition(12, $"_1")
+      .map { value =>
+        RealTimeInitialStateBootstrapBlock.awaitReleaseIfEnabled()
+        if (RealTimeInitialStateFailure.enabled) {
+          throw new RuntimeException("injected initial-state bootstrap failure")
+        }
+        value
+      }
+      .groupByKey(_._1)
+      .mapValues(_._2)
+  }
+
+  private def transformWithInitialState(
+      input: LowLatencyMemoryStream[(String, Int)],
+      values: Seq[(String, Long)]) = {
+    input.toDS()
+      .groupByKey(_._1)
+      .transformWithState(
+        new RealTimeInitialCountProcessor,
+        TimeMode.None(),
+        OutputMode.Update(),
+        initialState(values))
+  }
+
+  test("processes repeated keys within a long-running batch without a sort") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val (input, _) = createMemoryStream(numPartitions = 2)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeEagerCountProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1), ("a", 2), ("b", 1), ("a", 3)),
+        // The RTM batch remains open for five minutes. Seeing the final input here proves that
+        // TransformWithState processes rows individually rather than waiting for batch end.
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 1L), ("a", 2L), ("b", 1L), ("a", 3L)),
+        Execute { q =>
+          val operators = q.lastExecution.executedPlan.collect {
+            case t: TransformWithStateExec => t
+          }
+          assert(operators.size == 1, q.lastExecution.executedPlan)
+          assert(operators.head.isRealTimeMode)
+          assert(operators.head.requiredChildOrdering.forall(_.isEmpty))
+          assert(!operators.head.child.exists(_.isInstanceOf[SortExec]))
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("map state can emit an aggregate every other input") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val (input, clock) = createStringMemoryStream(numPartitions = 2)
+      val result = input.toDS()
+        .groupByKey(identity)
+        .transformWithState(
+          new RealTimeRunningCountStatefulProcessor(2L),
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, "a", "b"),
+        Execute { _ => waitForNextBatchToStart() },
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        CheckAnswerWithTimeout(60.seconds.toMillis),
+        Execute { _ => waitForNextBatchToStart() },
+        AddData(input, "c", "a", "a", "c"),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 2L), ("c", 2L)),
+        StopStream
+      )
+    }
+  }
+
+  test("expires TTL state while the RTM batch remains open") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS.key -> "1") {
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeTTLCountProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L)),
+        new ExternalAction {
+          override def runAction(): Unit = clock.advance(10001L)
+        },
+        // Processing another key runs the periodic TTL cleanup without closing the RTM batch.
+        AddData(input, ("b", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L), ("b", 1L)),
+        AddData(input, ("a", 2)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 1L), ("b", 1L), ("a", 1L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { q =>
+          val batch0 = q.recentProgress.find(_.batchId == 0).getOrElse {
+            fail(s"batch 0 progress was not retained: ${q.recentProgress.toSeq}")
+          }
+          assert(batch0.stateOperators.length == 1)
+          assert(batch0.stateOperators.head.customMetrics
+            .get("numValuesRemovedDueToTTLExpiry") > 0L)
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("cleans up expired TTL state at the end of an RTM batch") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS.key ->
+          "86400000") {
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeTTLCountProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L)),
+        advanceClock(clock, defaultTrigger.batchDurationMs + 10001L),
+        WaitUntilBatchProcessed(0),
+        Execute { q =>
+          val batch0 = q.recentProgress.find(_.batchId == 0).getOrElse {
+            fail(s"batch 0 progress was not retained: ${q.recentProgress.toSeq}")
+          }
+          assert(batch0.stateOperators.length == 1)
+          assert(batch0.stateOperators.head.numRowsTotal == 0L)
+          assert(batch0.stateOperators.head.customMetrics
+            .get("numValuesRemovedDueToTTLExpiry") == 1L)
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("expires value, map, and list state while the RTM batch remains open") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream[String](1)
+      val result = input.toDS()
+        .groupByKey(identity)
+        .transformWithState(
+          new MultiStatefulVariableTTLProcessor(TTLConfig(Duration.ofSeconds(10))),
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, "a", "b"),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L), ("b", 1L)),
+        new ExternalAction {
+          override def runAction(): Unit = clock.advance(10001L)
+        },
+        AddData(input, "c"),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 1L), ("b", 1L), ("c", 1L)),
+        AddData(input, "a"),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 1L), ("b", 1L), ("c", 1L), ("a", 1L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { q =>
+          val batch0 = q.recentProgress.find(_.batchId == 0).getOrElse {
+            fail(s"batch 0 progress was not retained: ${q.recentProgress.toSeq}")
+          }
+          assert(batch0.stateOperators.length == 1)
+          assert(batch0.stateOperators.head.customMetrics
+            .get("numValuesRemovedDueToTTLExpiry") >= 6L)
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("hides expired TTL state before the periodic cleanup scan") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS.key ->
+          "86400000") {
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeTTLCountProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L)),
+        new ExternalAction {
+          override def runAction(): Unit = clock.advance(10001L)
+        },
+        AddData(input, ("a", 2)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L), ("a", 1L)),
+        StopStream
+      )
+    }
+  }
+
+  test("cleans up every entry in an expired ListState") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS.key -> "1") {
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeListTTLProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1), ("b", 10)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 3L), ("b", 3L)),
+        new ExternalAction {
+          override def runAction(): Unit = clock.advance(10001L)
+        },
+        AddData(input, ("c", 20)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 3L), ("b", 3L), ("c", 3L)),
+        AddData(input, ("a", 30)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 3L), ("b", 3L), ("c", 3L), ("a", 3L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { q =>
+          val batch0 = q.recentProgress.find(_.batchId == 0).getOrElse {
+            fail(s"batch 0 progress was not retained: ${q.recentProgress.toSeq}")
+          }
+          assert(batch0.stateOperators.length == 1)
+          assert(batch0.stateOperators.head.customMetrics
+            .get("numValuesRemovedDueToTTLExpiry") >= 6L)
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("applies a shorter TTL after an RTM checkpoint restart") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STREAMING_TRANSFORM_WITH_STATE_REAL_TIME_MODE_TTL_EVICTION_INTERVAL_MS.key -> "1") {
+      withTempDir { checkpointDir =>
+        val (input, clock) = createMemoryStream(numPartitions = 1)
+        val checkpoint = checkpointDir.getCanonicalPath
+        val original = input.toDS()
+          .groupByKey(_._1)
+          .transformWithState(
+            new RealTimeTTLCountProcessor(Duration.ofSeconds(400)),
+            TimeMode.ProcessingTime(),
+            OutputMode.Update())
+
+        testStream(original, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          AddData(input, ("a", 1)),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L)),
+          advanceClock(clock),
+          WaitUntilBatchProcessed(0),
+          StopStream
+        )
+
+        val reduced = input.toDS()
+          .groupByKey(_._1)
+          .transformWithState(
+            new RealTimeTTLCountProcessor(Duration.ofSeconds(10)),
+            TimeMode.ProcessingTime(),
+            OutputMode.Update())
+
+        testStream(reduced, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          AddData(input, ("a", 2)),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 2L)),
+          new ExternalAction {
+            override def runAction(): Unit = clock.advance(10001L)
+          },
+          AddData(input, ("b", 1)),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 2L), ("b", 1L)),
+          AddData(input, ("a", 3)),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis, ("a", 2L), ("b", 1L), ("a", 1L)),
+          StopStream
+        )
+      }
+    }
+  }
+
+  test("RTM recovery ignores uncommitted current-batch operator metadata") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STATE_STORE_CHECKPOINT_FORMAT_VERSION.key -> "2") {
+      withTempDir { checkpointDir =>
+        val (input, clock) = createMemoryStream(numPartitions = 1)
+        val checkpoint = checkpointDir.getCanonicalPath
+        val result = input.toDS()
+          .groupByKey(_._1)
+          .transformWithState(
+            new RealTimeEagerCountProcessor,
+            TimeMode.None(),
+            OutputMode.Update())
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          AddData(input, ("a", 1)),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L)),
+          advanceClock(clock),
+          WaitUntilBatchProcessed(0),
+          StopStream
+        )
+
+        val hadoopConf = spark.sessionState.newHadoopConf()
+        val operatorStatePath = new Path(checkpoint, "state/0")
+        val committedMetadata = OperatorStateMetadataReader.createReader(
+          operatorStatePath,
+          hadoopConf,
+          version = 2,
+          batchId = 0).read().getOrElse {
+          fail("missing committed operator metadata for batch 0")
+        }.asInstanceOf[OperatorStateMetadataV2]
+
+        val invalidSchemaFile = checkpointDir.toPath.resolve("invalid-uncommitted-schema")
+        Files.write(invalidSchemaFile, "not a state schema".getBytes(StandardCharsets.UTF_8))
+        val uncommittedMetadata = committedMetadata.copy(
+          stateStoreInfo = committedMetadata.stateStoreInfo.map { storeInfo =>
+            storeInfo.copy(stateSchemaFilePaths = List(invalidSchemaFile.toString))
+          })
+        OperatorStateMetadataWriter.createWriter(
+          operatorStatePath,
+          hadoopConf,
+          version = 2,
+          currentBatchId = Some(1L)).write(uncommittedMetadata)
+
+        val uncommittedMetadataFile = checkpointDir.toPath.resolve("state/0/_metadata/v2/1")
+        assert(Files.isRegularFile(uncommittedMetadataFile))
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          AddData(input, ("a", 2)),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 2L)),
+          Execute { q =>
+            assert(q.lastExecution.currentBatchId == 1L)
+            assert(Files.isRegularFile(uncommittedMetadataFile))
+          },
+          advanceClock(clock),
+          WaitUntilBatchProcessed(1),
+          StopStream
+        )
+      }
+    }
+  }
+
+  test("runs transformWithState across multiple state store partitions") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "3") {
+      val (input, _) = createMemoryStream(numPartitions = 3)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimePartitionProcessor,
+          TimeMode.None(),
+          OutputMode.Update())
+      val sink = new ContinuousMemorySink()
+      val rows = (0 until 64).map(i => (s"key-$i", i))
+
+      testStream(result, OutputMode.Update(), sink = sink)(
+        StartStream(),
+        AddData(input, rows: _*),
+        Execute { _ =>
+          eventually(timeout(60.seconds)) {
+            assert(sink.allData.size == rows.size)
+            assert(sink.allData.map(_.getInt(1)).distinct.size > 1)
+          }
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("runs transformWithState after a union") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val leftInput = LowLatencyMemoryStream[(String, Int)](1)
+      val rightInput = LowLatencyMemoryStream[(String, Int)](1)
+
+      val result = leftInput.toDS()
+        .union(rightInput.toDS())
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeEagerCountProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(leftInput, ("a", 1)),
+        AddData(rightInput, ("a", 2), ("b", 1)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 1L), ("a", 2L), ("b", 1L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { _ => waitForNextBatchToStart() },
+        AddData(leftInput, ("a", 3)),
+        AddData(rightInput, ("b", 2)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("a", 1L),
+          ("a", 2L),
+          ("b", 1L),
+          ("a", 3L),
+          ("b", 2L)),
+        Execute { q =>
+          val operators = q.lastExecution.executedPlan.collect {
+            case transform: TransformWithStateExec => transform
+          }
+          assert(operators.size == 1, q.lastExecution.executedPlan)
+          assert(operators.head.isRealTimeMode)
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("runs transformWithState after real-time deduplication") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val (input, _) = createMemoryStream(numPartitions = 2)
+      val result = input.toDS()
+        .dropDuplicates()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeEagerCountProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1), ("b", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L), ("b", 1L)),
+        AddData(input, ("a", 1), ("a", 1), ("c", 1)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", 1L), ("b", 1L), ("c", 1L)),
+        StopStream
+      )
+    }
+  }
+
+  test("fires processing-time timers from a later row while the RTM batch remains open") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeProcessingTimerProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "data")),
+        new ExternalAction {
+          override def runAction(): Unit = clock.advance(10001L)
+        },
+        AddData(input, ("b", 1)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", "data"), ("b", "data"), ("a", "timer")),
+        StopStream
+      )
+    }
+  }
+
+  test("reuses one timer iterator task listener across RTM input rows") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val (input, _) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeTimerIteratorListenerProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1), ("a", 2), ("a", 3)),
+        // The first timer scan adds the reusable iterator's completion listener. Later scans
+        // refresh that iterator and must not add another listener.
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          (1, false),
+          (2, true),
+          (3, false)),
+        StopStream
+      )
+    }
+  }
+
+  test("expired timer receives the current RTM processing time") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      GlobalSingletonManualClock.reset()
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeProcessingTimerValueProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "data", 0L)),
+        new ExternalAction {
+          override def runAction(): Unit = clock.advance(10001L)
+        },
+        AddData(input, ("b", 1)),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("a", "data", 0L),
+          ("b", "data", 10001L),
+          ("a", "timer", 10001L)),
+        StopStream
+      )
+    }
+  }
+
+  test("fires remaining processing-time timers at the end of an RTM batch") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeProcessingTimerProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, ("a", 1)),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "data")),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", "data"), ("a", "timer")),
+        StopStream
+      )
+    }
+  }
+
+  test("processing time timers expire after multiple batches") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val (input, clock) = createStringMemoryStream()
+
+      val processor = new RTMStatefulProcessorWithProcTimeTimer(
+        defaultTrigger.batchDurationMs * 3)
+
+      val result = input
+        .toDS()
+        .groupByKey(x => x)
+        .transformWithState(processor, TimeMode.ProcessingTime(), OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, "a"),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "1")),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        // In real time mode, batches execute for a fixed amount of time. Wait for the next
+        // batch's tasks before advancing a manual clock again to avoid skipping its end time.
+        Execute { _ => waitForNextBatchToStart() },
+        advanceClock(clock),
+        WaitUntilBatchProcessed(1),
+        Execute { _ => waitForNextBatchToStart() },
+        advanceClock(clock),
+        WaitUntilBatchProcessed(2),
+        Execute { _ => waitForNextBatchToStart() },
+        advanceClock(clock, 1L),
+        AddData(input, "a"),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("a", "1"), ("a", "1"), ("a", "-1")),
+        StopStream
+      )
+    }
+  }
+
+  test("processing time timers single key multiple registered timers") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val (input, clock) = createStringMemoryStream()
+
+      val processor = new RTMStatefulProcessorWithProcTimeTimerWithMultipleTimers(30000)
+
+      val result = input
+        .toDS()
+        .groupByKey(x => x)
+        .transformWithState(processor, TimeMode.ProcessingTime(), OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, "a"),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "1")),
+        advanceClock(clock, 31001L),
+        AddData(input, "a"),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("a", "1"), ("a", "2"), ("a", "-1"), ("a", "-1")),
+        StopStream
+      )
+    }
+  }
+
+  test("processing time timers with timers from multiple keys") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val (input, clock) = createStringMemoryStream()
+
+      val processor = new RTMStatefulProcessorWithProcTimeTimer(30000)
+
+      val result = input
+        .toDS()
+        .groupByKey(x => x)
+        .transformWithState(processor, TimeMode.ProcessingTime(), OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, "a"),
+        AddData(input, "b"),
+        AddData(input, "c"),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "1"), ("b", "1"), ("c", "1")),
+        advanceClock(clock, 30001L),
+        AddData(input, "a"),
+        AddData(input, "b"),
+        AddData(input, "c"),
+        CheckAnswerRowsContainsWithTimeout(
+          60.seconds.toMillis,
+          ("a", "1"),
+          ("b", "1"),
+          ("c", "1"),
+          ("a", "-1"),
+          ("b", "2"),
+          ("c", "-1")
+        ),
+        StopStream
+      )
+    }
+  }
+
+  test("processing time timers with an integer key") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream[Int](2)
+      val result = input.toDS()
+        .groupByKey(identity)
+        .transformWithState(
+          new RTMStatefulProcessorWithProcTimeTimerInputInt(30000L),
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, 1),
+        CheckAnswerWithTimeout(60.seconds.toMillis, (1, 1L)),
+        advanceClock(clock, 30001L),
+        AddData(input, 1),
+        CheckAnswerRowsContainsWithTimeout(60.seconds.toMillis, (1, 1L), (1, -1L)),
+        StopStream
+      )
+    }
+  }
+
+  test("processing time timers with a product key") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream[(Int, String)](2)
+      val result = input.toDS()
+        .groupByKey(identity)
+        .transformWithState(
+          new RTMStatefulProcessorWithProcTimeTimerInputTuple(30000L),
+          TimeMode.ProcessingTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, (1, "a")),
+        CheckAnswerWithTimeout(60.seconds.toMillis, (1, "a", 1L)),
+        advanceClock(clock, 30001L),
+        AddData(input, (1, "a")),
+        CheckAnswerRowsContainsWithTimeout(60.seconds.toMillis, (1, "a", -1L)),
+        StopStream
+      )
+    }
+  }
+
+  test("processing time timers survive an RTM checkpoint restart") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      withTempDir { checkpointDir =>
+        val (input, clock) = createStringMemoryStream()
+        val processor = new RTMStatefulProcessorWithProcTimeTimer(
+          defaultTrigger.batchDurationMs + 1000L)
+        val result = input
+          .toDS()
+          .groupByKey(x => x)
+          .transformWithState(processor, TimeMode.ProcessingTime(), OutputMode.Update())
+        val checkpoint = checkpointDir.getCanonicalPath
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          AddData(input, "a"),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "1")),
+          advanceClock(clock),
+          WaitUntilBatchProcessed(0),
+          StopStream
+        )
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          Execute { _ => waitForNextBatchToStart() },
+          advanceClock(clock, 1001L),
+          AddData(input, "b"),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "-1"), ("b", "1")),
+          StopStream
+        )
+      }
+    }
+  }
+
+  test("uses fixed between-batch watermarks for incremental event-time timers") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream[(Timestamp, String)](1)
+      val result = input.toDF()
+        .select(col("_1").as("eventTime"), col("_2").as("key"))
+        .withWatermark("eventTime", "10 milliseconds")
+        .as[(Timestamp, String)]
+        .groupByKey(_._2)
+        .transformWithState(
+          new RealTimeEventTimerProcessor,
+          TimeMode.EventTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input,
+          (new Timestamp(100L), "a"),
+          (new Timestamp(200L), "a"),
+          (new Timestamp(150L), "a"),
+          (new Timestamp(300L), "a")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { _ => waitForNextBatchToStart() },
+        // Batch 1 uses batch 0's fixed eviction watermark (300 - 10 = 290). The first
+        // input row scans timers already below it without waiting for the batch to end.
+        AddData(input,
+          (new Timestamp(280L), "a"),
+          (new Timestamp(400L), "a")),
+        // The watermark does not move inside batch 1. A newly registered timer at 280 fires
+        // immediately against the same fixed 290 watermark.
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "data", 290L),
+          ("a", "timer", 290L),
+          ("a", "data", 290L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(1),
+        Execute { _ => waitForNextBatchToStart() },
+        // Batch 2 uses eviction watermark 390 and late-events watermark 290. The late 280 row
+        // still triggers the per-row timer scan even though it is not passed to the processor.
+        AddData(input, (new Timestamp(280L), "a")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "data", 290L),
+          ("a", "data", 290L),
+          ("a", "timer", 390L)),
+        // The watermark stays fixed while the accepted row is processed in the same batch.
+        AddData(input, (new Timestamp(500L), "a")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "data", 0L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "timer", 290L),
+          ("a", "data", 290L),
+          ("a", "data", 290L),
+          ("a", "timer", 390L),
+          ("a", "data", 390L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(2),
+        Execute { q =>
+          val batch2 = q.recentProgress.find(_.batchId == 2).getOrElse {
+            fail(s"batch 2 progress was not retained: ${q.recentProgress.toSeq}")
+          }
+          assert(batch2.stateOperators.length == 1)
+          assert(batch2.stateOperators.head.numRowsDroppedByWatermark == 1L)
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("fires a newly expired event-time timer from its RTM input row") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream[(Timestamp, String)](1)
+      val result = input.toDF()
+        .select(col("_1").as("eventTime"), col("_2").as("key"))
+        .withWatermark("eventTime", "10 milliseconds")
+        .as[(Timestamp, String)]
+        .groupByKey(_._2)
+        .transformWithState(
+          new RealTimeEventTimerProcessor,
+          TimeMode.EventTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, (new Timestamp(300L), "a")),
+        CheckAnswerWithTimeout(10.seconds.toMillis, ("a", "data", 0L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { _ => waitForNextBatchToStart() },
+        // Batch 1's fixed eviction watermark is 290, so the timer registered for this row is
+        // scanned immediately after the row is processed.
+        AddData(input, (new Timestamp(280L), "a")),
+        CheckAnswerWithTimeout(
+          10.seconds.toMillis,
+          ("a", "data", 0L),
+          ("a", "data", 290L),
+          ("a", "timer", 290L)),
+        StopStream
+      )
+    }
+  }
+
+  test("fires event-time timers in the final scan of an empty RTM batch") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream[(Timestamp, String)](1)
+      val result = input.toDF()
+        .select(col("_1").as("eventTime"), col("_2").as("key"))
+        .withWatermark("eventTime", "10 milliseconds")
+        .as[(Timestamp, String)]
+        .groupByKey(_._2)
+        .transformWithState(
+          new RealTimeEventTimerProcessor,
+          TimeMode.EventTime(),
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input,
+          (new Timestamp(100L), "expired"),
+          (new Timestamp(300L), "watermark")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("expired", "data", 0L),
+          ("watermark", "data", 0L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { _ => waitForNextBatchToStart() },
+        // Batch 1 has no input rows. Its final scan uses batch 0's fixed eviction watermark.
+        advanceClock(clock),
+        WaitUntilBatchProcessed(1),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          ("expired", "data", 0L),
+          ("watermark", "data", 0L),
+          ("expired", "timer", 290L)),
+        Execute { q =>
+          val batch1 = q.recentProgress.find(_.batchId == 1).getOrElse {
+            fail(s"batch 1 progress was not retained: ${q.recentProgress.toSeq}")
+          }
+          assert(batch1.numInputRows == 0L)
+        },
+        StopStream
+      )
+    }
+  }
+
+  test("recovers event-time timers and fires them from the next row after restart") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      withTempDir { checkpointDir =>
+        val clock = new GlobalSingletonManualClock()
+        LowLatencyClock.setClock(clock)
+        val input = LowLatencyMemoryStream[(Timestamp, String)](1)
+        val result = input.toDF()
+          .select(col("_1").as("eventTime"), col("_2").as("key"))
+          .withWatermark("eventTime", "10 milliseconds")
+          .as[(Timestamp, String)]
+          .groupByKey(_._2)
+          .transformWithState(
+            new RealTimeEventTimerProcessor,
+            TimeMode.EventTime(),
+            OutputMode.Update())
+        val checkpoint = checkpointDir.getCanonicalPath
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          AddData(input,
+            (new Timestamp(100L), "expired"),
+            (new Timestamp(300L), "watermark")),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis,
+            ("expired", "data", 0L),
+            ("watermark", "data", 0L)),
+          advanceClock(clock),
+          WaitUntilBatchProcessed(0),
+          StopStream
+        )
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(checkpointLocation = checkpoint),
+          Execute { _ => waitForNextBatchToStart() },
+          // The recovered batch watermark is 300 - 10 = 290. The next input row scans the
+          // recovered timers against that fixed watermark.
+          AddData(input, (new Timestamp(400L), "trigger")),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis,
+            ("trigger", "data", 290L),
+            ("expired", "timer", 290L)),
+          StopStream
+        )
+      }
+    }
+  }
+
+  test("supports an output event-time column in RTM") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream.singlePartition[(Timestamp, String)]
+      val result = input.toDF()
+        .select(col("_1").as("eventTime"), col("_2").as("key"))
+        .withWatermark("eventTime", "1 minute")
+        .as[(Timestamp, String)]
+        .groupByKey(_._2)
+        .transformWithState[RealTimeEventTimeOutputRow](
+          new RealTimeEventTimeOutputProcessor,
+          "outputEventTime",
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input,
+          (new Timestamp(1000000L), "a"),
+          (new Timestamp(2000000L), "a")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          RealTimeEventTimeOutputRow("a", new Timestamp(1000000L), 1L),
+          RealTimeEventTimeOutputRow("a", new Timestamp(2000000L), 2L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { _ => waitForNextBatchToStart() },
+        AddData(input, (new Timestamp(3000000L), "a")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          RealTimeEventTimeOutputRow("a", new Timestamp(1000000L), 1L),
+          RealTimeEventTimeOutputRow("a", new Timestamp(2000000L), 2L),
+          RealTimeEventTimeOutputRow("a", new Timestamp(3000000L), 3L)),
+        StopStream
+      )
+    }
+  }
+
+  test("uses the fixed prior-batch watermark for output event-time validation") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream.singlePartition[(Timestamp, String)]
+      val result = input.toDF()
+        .select(col("_1").as("eventTime"), col("_2").as("key"))
+        .withWatermark("eventTime", "10 milliseconds")
+        .as[(Timestamp, String)]
+        .groupByKey(_._2)
+        .transformWithState[RealTimeEventTimeOutputRow](
+          new RealTimeEventTimeOutputProcessor(Some(new Timestamp(1L))),
+          "outputEventTime",
+          OutputMode.Update())
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(),
+        AddData(input, (new Timestamp(300L), "a")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          RealTimeEventTimeOutputRow("a", new Timestamp(1L), 1L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(0),
+        Execute { _ => waitForNextBatchToStart() },
+        AddData(input, (new Timestamp(400L), "a")),
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis,
+          RealTimeEventTimeOutputRow("a", new Timestamp(1L), 1L),
+          RealTimeEventTimeOutputRow("a", new Timestamp(1L), 2L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(1),
+        Execute { _ => waitForNextBatchToStart() },
+        // Batch 2's late-events watermark is fixed at 290. Its accepted input would emit an
+        // output timestamp of 1, which must be rejected against that same fixed watermark.
+        AddData(input, (new Timestamp(500L), "a")),
+        ExpectFailure[SparkRuntimeException] { error =>
+          checkError(
+            error.asInstanceOf[SparkThrowable],
+            "EMITTING_ROWS_OLDER_THAN_WATERMARK_NOT_ALLOWED",
+            parameters = Map(
+              "currentWatermark" -> "290",
+              "emittedRowEventTime" -> "1000"))
+        }
+      )
+    }
+  }
+
+  test("processing-time timer registered from initial state uses the batch timestamp") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      GlobalSingletonManualClock.reset()
+      val (input, executorClock) = createStringMemoryStream(numPartitions = 1)
+      val driverClock = new StreamManualClock(100000L)
+      val result = input.toDS()
+        .groupByKey(identity)
+        .transformWithState(
+          new RealTimeInitialStateProcTimerWithExpiryProcessor,
+          TimeMode.ProcessingTime(),
+          OutputMode.Update(),
+          Seq("a").toDS().groupByKey(identity))
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        StartStream(triggerClock = driverClock),
+        WaitUntilBatchProcessed(0),
+        Execute { _ => waitForNextBatchToStart() },
+        // The initial-state timer is based on batch 0's driver timestamp (100000), not the
+        // executor clock (0), so advancing the executor by only the timer delay must not fire it.
+        advanceClock(executorClock, 5001L),
+        Execute { _ => input.addData(Seq("b")) },
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("b", "1")),
+        advanceClock(executorClock, 100000L),
+        Execute { _ => input.addData(Seq("c")) },
+        CheckAnswerWithTimeout(
+          60.seconds.toMillis, ("b", "1"), ("c", "1"), ("a", "105000")),
+        StopStream
+      )
+    }
+  }
+
+  test("initial-state bootstrap batch does not use pipelined shuffle") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      val (input, _) = createMemoryStream(numPartitions = 2)
+      val result = transformWithInitialState(input, Seq("a" -> 2L, "b" -> 5L))
+
+      RealTimeInitialStateBootstrapBlock.enable()
+      try {
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          StartStream(),
+          Execute { q =>
+            try {
+              assert(RealTimeInitialStateBootstrapBlock.awaitTaskStart())
+              val execution = q.lastExecution
+              assert(execution.currentBatchId == 0L)
+              val plan = execution.executedPlan
+              val scans = plan.collect { case scan: RealTimeStreamScanExec => scan }
+              assert(scans.nonEmpty)
+              assert(scans.forall(_.batchDurationMs == 0L))
+
+              val shuffles = plan.collect { case exchange: ShuffleExchangeExec => exchange }
+              assert(shuffles.nonEmpty)
+              assert(shuffles.forall(!_.pipelined))
+            } finally {
+              RealTimeInitialStateBootstrapBlock.disable()
+            }
+          },
+          WaitUntilBatchProcessed(0),
+          StopStream
+        )
+      } finally {
+        RealTimeInitialStateBootstrapBlock.disable()
+      }
+    }
+  }
+
+  Seq(
+    "non-empty" -> Seq("a" -> 2L, "b" -> 5L),
+    "empty" -> Seq.empty[(String, Long)]
+  ).foreach { case (description, initialValues) =>
+    test(s"hydrates $description initial state in a finite batch and recovers it") {
+      withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+        withTempDir { checkpointDir =>
+          val (input, clock) = createMemoryStream(numPartitions = 2)
+          val result = transformWithInitialState(input, initialValues)
+          val initialCount = initialValues.toMap.getOrElse("a", 0L)
+
+          testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+            // Input available when the query starts must wait until initial state is durable.
+            AddData(input, ("a", 1)),
+            StartStream(checkpointLocation = checkpointDir.getCanonicalPath),
+            WaitUntilBatchProcessed(0),
+            Execute { q =>
+              val batch0 = q.recentProgress.find(_.batchId == 0).getOrElse {
+                fail(s"batch 0 progress was not retained: ${q.recentProgress.toSeq}")
+              }
+              assert(batch0.numInputRows == 0)
+              assert(batch0.stateOperators.length == 1)
+              assert(batch0.stateOperators.head.numRowsTotal == initialValues.size)
+            },
+            CheckAnswerWithTimeout(60.seconds.toMillis, ("a", initialCount + 1L)),
+            Execute { q =>
+              val plan = q.lastExecution.executedPlan
+              val scans = plan.collect { case scan: RealTimeStreamScanExec => scan }
+              assert(scans.nonEmpty)
+              assert(scans.forall(_.batchDurationMs == defaultTrigger.batchDurationMs))
+              val streamingShuffles = plan.collect {
+                case exchange: ShuffleExchangeExec if exchange.exists {
+                    case _: RealTimeStreamScanExec => true
+                    case _ => false
+                  } => exchange
+              }
+              assert(streamingShuffles.nonEmpty)
+              assert(streamingShuffles.forall(_.pipelined))
+            },
+            advanceClock(clock),
+            WaitUntilBatchProcessed(1),
+            StopStream
+          )
+
+          testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+            StartStream(checkpointLocation = checkpointDir.getCanonicalPath),
+            AddData(input, ("a", 2)),
+            CheckAnswerWithTimeout(60.seconds.toMillis, ("a", initialCount + 2L)),
+            advanceClock(clock),
+            WaitUntilBatchProcessed(2),
+            StopStream
+          )
+        }
+      }
+    }
+  }
+
+  test("hydrates event-time initial state before the first RTM input batch") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val clock = new GlobalSingletonManualClock()
+      LowLatencyClock.setClock(clock)
+      val input = LowLatencyMemoryStream[(Timestamp, String)](1)
+      val result = input.toDF()
+        .select(col("_1").as("eventTime"), col("_2").as("key"))
+        .withWatermark("eventTime", "10 milliseconds")
+        .as[(Timestamp, String)]
+        .groupByKey(_._2)
+        .transformWithState(
+          new RealTimeEventInitialCountProcessor,
+          TimeMode.EventTime(),
+          OutputMode.Update(),
+          initialState(Seq("a" -> 2L)))
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        AddData(input, (new Timestamp(100L), "a")),
+        StartStream(),
+        WaitUntilBatchProcessed(0),
+        Execute { q =>
+          val batch0 = q.recentProgress.find(_.batchId == 0).getOrElse {
+            fail(s"batch 0 progress was not retained: ${q.recentProgress.toSeq}")
+          }
+          assert(batch0.numInputRows == 0)
+          assert(batch0.stateOperators.length == 1)
+          assert(batch0.stateOperators.head.numRowsTotal == 1)
+        },
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 3L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(1),
+        StopStream
+      )
+    }
+  }
+
+  test("processes non-contiguous duplicate initial-state keys without sorting") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      val (input, clock) = createMemoryStream(numPartitions = 1)
+      // A single input partition preserves a, b, a through the one-partition hash exchange.
+      val duplicateInitialState = Seq("a" -> 1L, "b" -> 5L, "a" -> 2L)
+        .toDS()
+        .coalesce(1)
+        .groupByKey(_._1)
+        .mapValues(_._2)
+      val result = input.toDS()
+        .groupByKey(_._1)
+        .transformWithState(
+          new RealTimeInitialCountProcessor,
+          TimeMode.None(),
+          OutputMode.Update(),
+          duplicateInitialState)
+
+      testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+        AddData(input, ("a", 1)),
+        StartStream(),
+        WaitUntilBatchProcessed(0),
+        CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 4L)),
+        advanceClock(clock),
+        WaitUntilBatchProcessed(1),
+        StopStream
+      )
+    }
+  }
+
+  test("retries initial-state bootstrap from batch 0 after failure") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "2") {
+      withTempDir { checkpointDir =>
+        val (input, clock) = createMemoryStream(numPartitions = 2)
+        val result = transformWithInitialState(input, Seq("a" -> 2L, "b" -> 5L))
+        val offsetFile = new java.io.File(checkpointDir, "offsets/0")
+
+        try {
+          RealTimeInitialStateFailure.enabled = true
+          testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+            StartStream(checkpointLocation = checkpointDir.getCanonicalPath),
+            ExpectFailure[SparkException] { error =>
+              assert(error.getMessage.contains("injected initial-state bootstrap failure"))
+            }
+          )
+          assert(!offsetFile.exists())
+
+          RealTimeInitialStateFailure.enabled = false
+          testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+            AddData(input, ("a", 1)),
+            StartStream(checkpointLocation = checkpointDir.getCanonicalPath),
+            WaitUntilBatchProcessed(0),
+            CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 3L)),
+            advanceClock(clock),
+            WaitUntilBatchProcessed(1),
+            StopStream
+          )
+        } finally {
+          RealTimeInitialStateFailure.enabled = false
+        }
+      }
+    }
+  }
+
+  Seq(true, false).foreach { changelogCheckpointingEnabled =>
+    test(s"recovers transformWithState state with RocksDB changelog checkpointing " +
+        s"enabled=$changelogCheckpointingEnabled") {
+      val changelogKey =
+        s"${RocksDBConf.ROCKSDB_SQL_CONF_NAME_PREFIX}.changelogCheckpointing.enabled"
+      withSQLConf(
+          SQLConf.SHUFFLE_PARTITIONS.key -> "2",
+          changelogKey -> changelogCheckpointingEnabled.toString) {
+        withTempDir { checkpointDir =>
+          val (input, clock) = createMemoryStream(numPartitions = 2)
+          val result = input.toDS()
+            .groupByKey(_._1)
+            .transformWithState(
+              new RealTimeEagerCountProcessor,
+              TimeMode.ProcessingTime(),
+              OutputMode.Update())
+
+          testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+            StartStream(checkpointLocation = checkpointDir.getCanonicalPath),
+            AddData(input, ("a", 1), ("a", 2)),
+            CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 1L), ("a", 2L)),
+            Execute { q =>
+              assert(q.sparkSessionForStream.conf.get(changelogKey) ==
+                changelogCheckpointingEnabled.toString)
+            },
+            advanceClock(clock),
+            WaitUntilBatchProcessed(0),
+            StopStream
+          )
+
+          testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+            StartStream(checkpointLocation = checkpointDir.getCanonicalPath),
+            AddData(input, ("a", 3), ("b", 1)),
+            CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 3L), ("b", 1L)),
+            advanceClock(clock),
+            WaitUntilBatchProcessed(1),
+            StopStream
+          )
+        }
+      }
+    }
+  }
+
+  test("keeps transformWithState state across MBM to RTM to MBM restarts") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STATE_STORE_CHECKPOINT_FORMAT_VERSION.key -> "2") {
+      withTempDir { checkpointDir =>
+        val (input, clock) = createMemoryStream(numPartitions = 1)
+        val result = input.toDS()
+          .groupByKey(_._1)
+          .transformWithState(
+            new RealTimeEagerCountProcessor,
+            TimeMode.None(),
+            OutputMode.Update())
+        val checkpoint = checkpointDir.getCanonicalPath
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          AddData(input, ("a", 1), ("a", 2)),
+          StartStream(
+            trigger = Trigger.ProcessingTime(1000),
+            checkpointLocation = checkpoint),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", 2L)),
+          WaitUntilBatchProcessed(0),
+          Execute { q =>
+            val operators = q.lastExecution.executedPlan.collect {
+              case transform: TransformWithStateExec => transform
+            }
+            assert(operators.size == 1)
+            assert(!operators.head.isRealTimeMode)
+          },
+          StopStream,
+          StartStream(trigger = defaultTrigger, checkpointLocation = checkpoint),
+          AddData(input, ("a", 3), ("b", 1)),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis, ("a", 2L), ("a", 3L), ("b", 1L)),
+          Execute { q =>
+            val operators = q.lastExecution.executedPlan.collect {
+              case transform: TransformWithStateExec => transform
+            }
+            assert(operators.size == 1)
+            assert(operators.head.isRealTimeMode)
+          },
+          advanceClock(clock),
+          WaitUntilBatchProcessed(1),
+          StopStream,
+          AddData(input, ("a", 4), ("b", 2)),
+          StartStream(
+            trigger = Trigger.ProcessingTime(1000),
+            checkpointLocation = checkpoint),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis,
+            ("a", 2L), ("a", 3L), ("b", 1L), ("a", 4L), ("b", 2L)),
+          WaitUntilBatchProcessed(2),
+          Execute { q =>
+            val operators = q.lastExecution.executedPlan.collect {
+              case transform: TransformWithStateExec => transform
+            }
+            assert(operators.size == 1)
+            assert(!operators.head.isRealTimeMode)
+          },
+          StopStream
+        )
+      }
+    }
+  }
+
+  test("keeps MapState TTL and processing-time timers across MBM to RTM to MBM restarts") {
+    withSQLConf(
+        SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+        SQLConf.STATE_STORE_CHECKPOINT_FORMAT_VERSION.key -> "2") {
+      withTempDir { checkpointDir =>
+        GlobalSingletonManualClock.reset()
+        val (input, clock) = createMemoryStream(numPartitions = 1)
+        val result = input.toDS()
+          .groupByKey(_._1)
+          .transformWithState(
+            new RealTimeMapTTLAndTimerProcessor,
+            TimeMode.ProcessingTime(),
+            OutputMode.Update())
+        val checkpoint = checkpointDir.getCanonicalPath
+        val timerDelayAcrossRtmBoundary = (defaultTrigger.batchDurationMs + 1000L).toInt
+
+        testStream(result, OutputMode.Update(), sink = new ContinuousMemorySink())(
+          AddData(input, ("a", 10000)),
+          StartStream(
+            trigger = Trigger.ProcessingTime(1000),
+            triggerClock = clock,
+            checkpointLocation = checkpoint),
+          CheckAnswerWithTimeout(60.seconds.toMillis, ("a", "data", 1L)),
+          WaitUntilBatchProcessed(0),
+          StopStream,
+          StartStream(trigger = defaultTrigger, checkpointLocation = checkpoint),
+          Execute { _ => waitForNextBatchToStart() },
+          advanceClock(clock, 10001L),
+          AddData(input, ("a", timerDelayAcrossRtmBoundary)),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis,
+            ("a", "data", 1L),
+            ("a", "data", 2L),
+            ("a", "timer", 10000L)),
+          advanceClock(clock),
+          WaitUntilBatchProcessed(1),
+          StopStream,
+          advanceClock(clock, 1001L),
+          AddData(input, ("a", 10000)),
+          StartStream(
+            trigger = Trigger.ProcessingTime(1000),
+            triggerClock = clock,
+            checkpointLocation = checkpoint),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis,
+            ("a", "data", 1L),
+            ("a", "data", 2L),
+            ("a", "timer", 10000L),
+            ("a", "data", 3L),
+            ("a", "timer", 311001L)),
+          WaitUntilBatchProcessed(2),
+          StopStream,
+          advanceClock(clock, Duration.ofMinutes(10).toMillis + 1L),
+          AddData(input, ("a", 10000)),
+          StartStream(
+            trigger = Trigger.ProcessingTime(1000),
+            triggerClock = clock,
+            checkpointLocation = checkpoint),
+          CheckAnswerWithTimeout(
+            60.seconds.toMillis,
+            ("a", "data", 1L),
+            ("a", "data", 2L),
+            ("a", "timer", 10000L),
+            ("a", "data", 3L),
+            ("a", "timer", 311001L),
+            ("a", "data", 1L),
+            ("a", "timer", 321002L)),
+          WaitUntilBatchProcessed(3),
+          StopStream
+        )
+      }
+    }
+  }
+}
+
+class RealTimeTransformWithStateSuiteWithRowChecksum
+  extends RealTimeTransformWithStateSuite with EnableStateStoreRowChecksum
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeDefaultConfsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeDefaultConfsSuite.scala
index 5b84f4cfa03d1..65fb2c09a53bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeDefaultConfsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeDefaultConfsSuite.scala
@@ -167,7 +167,7 @@ class StreamRealTimeModeDefaultConfsSuite extends StreamRealTimeModeSuiteBase {
 
   test("switching an existing v1 checkpoint to Real-Time Mode fails fast") {
     // An existing v1 checkpoint carries no explicit config, so the pre-flight cannot see it; the
-    // initializeExecution fail-fast catches it instead, from the resolved commit log version.
+    // initializeExecution fail-fast catches it instead, from the resolved state-store format.
     // Resolution keeps an existing checkpoint at the version it was created with, so rather than
     // silently running at v1 (where a re-executed batch can reuse the failed batch's state file
     // names and lose data) the query is rejected at start. The rejection is unconditional -- a
@@ -221,9 +221,9 @@ class StreamRealTimeModeDefaultConfsSuite extends StreamRealTimeModeSuiteBase {
   }
 
   test("a fresh Real-Time Mode checkpoint is not rejected") {
-    // A fresh checkpoint takes v2 from the Real-Time Mode defaults, so its resolved commit log is
-    // v2 and the rejection does not apply. Only a v1 commit log -- an existing v1 checkpoint, or an
-    // explicit v1 config without the escape hatch -- is rejected.
+    // A fresh checkpoint takes state-store format v2 from the Real-Time Mode defaults, so the
+    // rejection does not apply. Only a resolved state-store format v1 -- from an existing
+    // incompatible checkpoint or an explicit v1 config without the escape hatch -- is rejected.
     withTempDir { checkpointDir =>
       val inputData = LowLatencyMemoryStream[Int]
       testStream(inputData.toDS(), OutputMode.Update, Map.empty, new ContinuousMemorySink())(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeSuite.scala
index 52b05b6f4fb5f..5ca440b244125 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeSuite.scala
@@ -33,7 +33,8 @@ import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExcha
 import org.apache.spark.sql.execution.streaming.RealTimeTrigger
 import org.apache.spark.sql.execution.streaming.runtime.{MemoryStream, StreamExecution}
 import org.apache.spark.sql.execution.streaming.sources.{ContinuousMemorySink, LowLatencyMemoryStream}
-import org.apache.spark.sql.execution.streaming.state.{FailureInjectionCheckpointFileManager, FailureInjectionFileSystem}
+import org.apache.spark.sql.execution.streaming.state.{FailureInjectionCheckpointFileManager,
+  FailureInjectionFileSystem, RocksDBStateStoreProvider}
 import org.apache.spark.sql.functions.{broadcast, concat, lit, udf}
 import org.apache.spark.sql.internal.SQLConf
 
@@ -468,6 +469,57 @@ class StreamRealTimeModeWithManualClockSuite extends StreamRealTimeModeManualClo
     )
   }
 
+  test("transformWithState writes batch 0 metadata only after the RTM offset WAL") {
+    withSQLConf(
+      SQLConf.STREAMING_CHECKPOINT_FILE_MANAGER_CLASS.parent.key ->
+        classOf[FailureInjectionCheckpointFileManager].getName,
+      SQLConf.STATE_STORE_PROVIDER_CLASS.key -> classOf[RocksDBStateStoreProvider].getName,
+      SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+      withTempDir { checkpointDir =>
+        val injectionState = FailureInjectionFileSystem.registerTempPath(checkpointDir.getPath)
+        try {
+          val inputData = LowLatencyMemoryStream[String](1)
+          val result = inputData.toDS()
+            .groupByKey(value => value)
+            .transformWithState(
+              new RunningCountStatefulProcessor,
+              TimeMode.ProcessingTime(),
+              OutputMode.Update())
+          val metadataFile = new java.io.File(
+            checkpointDir, "state/0/_metadata/v2/0")
+          val stateSchemaDir = new java.io.File(
+            checkpointDir, "state/0/_stateSchema/default")
+
+          injectionState.failureCreateAtomicRegex = Seq(".*/offsets/0")
+          testStream(result, OutputMode.Update, Map.empty, new ContinuousMemorySink())(
+            AddData(inputData, "a"),
+            StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+            CheckAnswerWithTimeout(60000, ("a", "1")),
+            Execute { _ =>
+              assert(Option(stateSchemaDir.listFiles()).exists(_.nonEmpty))
+              assert(!metadataFile.exists())
+            },
+            advanceRealTimeClock,
+            ExpectFailure[IOException]()
+          )
+          assert(!metadataFile.exists())
+
+          injectionState.failureCreateAtomicRegex = Seq.empty
+          testStream(result, OutputMode.Update, Map.empty, new ContinuousMemorySink())(
+            StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
+            CheckAnswerWithTimeout(60000, ("a", "1")),
+            advanceRealTimeClock,
+            WaitUntilBatchProcessed(0),
+            StopStream
+          )
+          assert(metadataFile.exists())
+        } finally {
+          FailureInjectionFileSystem.removePathFromTempToInjectionState(checkpointDir.getPath)
+        }
+      }
+    }
+  }
+
   // ========================================================================================
   // Pipelined (streaming) shuffle: a stateful/repartition Real-Time Mode query whose shuffle is a
   // PipelinedShuffleDependency, so the producer (source scan) and consumer stages are co-scheduled