diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 4e68d871b81a..3ab6b49eb04a 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -2249,6 +2249,10 @@ async fn collect_left_input(
         memory_counter: _,
     } = state;
 
+    // Only time hash-table construction, not the `try_fold` above (that's the
+    // child's own compute).
+    let _build_timer = metrics.build_time.timer();
+
     // Compute bounds
     let mut bounds = match bounds_accumulators {
         Some(accumulators) if num_rows > 0 => {
diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs
index 686939537e73..f8199d361335 100644
--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs
@@ -660,7 +660,6 @@ impl HashJoinStream {
         &mut self,
         cx: &mut std::task::Context<'_>,
     ) -> Poll<Result<StatefulStreamResult<Option<RecordBatch>>>> {
-        let build_timer = self.join_metrics.build_time.timer();
         // build hash table from left (build) side, if not yet done
         let left_data = ready!(
             self.build_side
@@ -668,7 +667,6 @@ impl HashJoinStream {
                 .left_fut
                 .get_shared(cx)
         )?;
-        build_timer.done();
 
         // Note: For null-aware anti join, we need to check the probe side (right) for NULLs,
         // not the build side (left). The probe-side NULL check happens during process_probe_batch.