diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 2f1ea4f219..27e20b3262 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -150,6 +150,8 @@ jobs: - tests::signer::v0::block_proposal_timeout - tests::signer::v0::rejected_blocks_count_towards_miner_validity - tests::signer::v0::allow_reorg_within_first_proposal_burn_block_timing_secs + - tests::signer::v0::prev_miner_extends_if_incoming_miner_fails_to_mine + - tests::signer::v0::prev_miner_will_not_extend_if_incoming_miner_mines - tests::nakamoto_integrations::burn_ops_integration_test - tests::nakamoto_integrations::check_block_heights - tests::nakamoto_integrations::clarity_burn_state diff --git a/CHANGELOG.md b/CHANGELOG.md index 1644446dd7..04f77d6d0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,13 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE ### Added - Add miner configuration option `tenure_extend_cost_threshold` to specify the percentage of the tenure budget that must be spent before a time-based tenure extend is attempted +- Add miner configuration option `tenure_extend_wait_timeout_ms` to specify the time to wait to try to continue a tenure if a BlockFound is expected ### Changed - Miner will include other transactions in blocks with tenure extend transactions (#5760) - Miner will not issue a tenure extend until at least half of the block budget has been spent (#5757) +- Miner will issue a tenure extend if the incoming miner has failed to produce a block (#5729) ### Fixed diff --git a/stacks-signer/CHANGELOG.md b/stacks-signer/CHANGELOG.md index df30e0d0db..ead5f9c067 100644 --- a/stacks-signer/CHANGELOG.md +++ b/stacks-signer/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE `StackerDB` messages, it logs `INFO` messages. Other interactions with the `stacks-node` behave normally (e.g., submitting validation requests, submitting finished blocks). A dry run signer will error out if the supplied key is actually a registered signer. +- Reduce default value of `block_proposal_timeout_ms` to 120_000 ## [3.1.0.0.4.0] diff --git a/stacks-signer/src/config.rs b/stacks-signer/src/config.rs index 29ee35c961..556920a8ba 100644 --- a/stacks-signer/src/config.rs +++ b/stacks-signer/src/config.rs @@ -35,7 +35,7 @@ use stacks_common::util::hash::Hash160; use crate::client::SignerSlotID; const EVENT_TIMEOUT_MS: u64 = 5000; -const BLOCK_PROPOSAL_TIMEOUT_MS: u64 = 600_000; +const BLOCK_PROPOSAL_TIMEOUT_MS: u64 = 120_000; const BLOCK_PROPOSAL_VALIDATION_TIMEOUT_MS: u64 = 120_000; const DEFAULT_FIRST_PROPOSAL_BURN_BLOCK_TIMING_SECS: u64 = 60; const DEFAULT_TENURE_LAST_BLOCK_PROPOSAL_TIMEOUT_SECS: u64 = 30; diff --git a/stackslib/src/config/mod.rs b/stackslib/src/config/mod.rs index 2e494cacef..27ef80bf44 100644 --- a/stackslib/src/config/mod.rs +++ b/stackslib/src/config/mod.rs @@ -114,6 +114,8 @@ const DEFAULT_TENURE_COST_LIMIT_PER_BLOCK_PERCENTAGE: u8 = 25; /// see if we need to extend the ongoing tenure (e.g. because the current /// sortition is empty or invalid). const DEFAULT_TENURE_EXTEND_POLL_SECS: u64 = 1; +/// Default number of millis to wait to try to continue a tenure if a BlockFound is expected +const DEFAULT_TENURE_EXTEND_WAIT_MS: u64 = 120_000; /// Default duration to wait before attempting to issue a tenure extend. /// This should be greater than the signers' timeout. This is used for issuing /// fallback tenure extends @@ -2177,9 +2179,11 @@ pub struct MinerConfig { pub block_commit_delay: Duration, /// The percentage of the remaining tenure cost limit to consume each block. pub tenure_cost_limit_per_block_percentage: Option, - /// The number of seconds to wait in-between polling the sortition DB to see if we need to + /// Duration to wait in-between polling the sortition DB to see if we need to /// extend the ongoing tenure (e.g. because the current sortition is empty or invalid). - pub tenure_extend_poll_secs: Duration, + pub tenure_extend_poll_timeout: Duration, + /// Duration to wait to try to continue a tenure if a BlockFound is expected + pub tenure_extend_wait_timeout: Duration, /// Duration to wait before attempting to issue a tenure extend pub tenure_timeout: Duration, /// Percentage of block budget that must be used before attempting a time-based tenure extend @@ -2220,7 +2224,8 @@ impl Default for MinerConfig { tenure_cost_limit_per_block_percentage: Some( DEFAULT_TENURE_COST_LIMIT_PER_BLOCK_PERCENTAGE, ), - tenure_extend_poll_secs: Duration::from_secs(DEFAULT_TENURE_EXTEND_POLL_SECS), + tenure_extend_poll_timeout: Duration::from_secs(DEFAULT_TENURE_EXTEND_POLL_SECS), + tenure_extend_wait_timeout: Duration::from_millis(DEFAULT_TENURE_EXTEND_WAIT_MS), tenure_timeout: Duration::from_secs(DEFAULT_TENURE_TIMEOUT_SECS), tenure_extend_cost_threshold: DEFAULT_TENURE_EXTEND_COST_THRESHOLD, } @@ -2618,6 +2623,7 @@ pub struct MinerConfigFile { pub block_commit_delay_ms: Option, pub tenure_cost_limit_per_block_percentage: Option, pub tenure_extend_poll_secs: Option, + pub tenure_extend_wait_timeout_ms: Option, pub tenure_timeout_secs: Option, pub tenure_extend_cost_threshold: Option, } @@ -2760,7 +2766,8 @@ impl MinerConfigFile { subsequent_rejection_pause_ms: self.subsequent_rejection_pause_ms.unwrap_or(miner_default_config.subsequent_rejection_pause_ms), block_commit_delay: self.block_commit_delay_ms.map(Duration::from_millis).unwrap_or(miner_default_config.block_commit_delay), tenure_cost_limit_per_block_percentage, - tenure_extend_poll_secs: self.tenure_extend_poll_secs.map(Duration::from_secs).unwrap_or(miner_default_config.tenure_extend_poll_secs), + tenure_extend_poll_timeout: self.tenure_extend_poll_secs.map(Duration::from_secs).unwrap_or(miner_default_config.tenure_extend_poll_timeout), + tenure_extend_wait_timeout: self.tenure_extend_wait_timeout_ms.map(Duration::from_millis).unwrap_or(miner_default_config.tenure_extend_wait_timeout), tenure_timeout: self.tenure_timeout_secs.map(Duration::from_secs).unwrap_or(miner_default_config.tenure_timeout), tenure_extend_cost_threshold: self.tenure_extend_cost_threshold.unwrap_or(miner_default_config.tenure_extend_cost_threshold), }) diff --git a/testnet/stacks-node/src/nakamoto_node/relayer.rs b/testnet/stacks-node/src/nakamoto_node/relayer.rs index 2cbc37acff..86ae5a4de4 100644 --- a/testnet/stacks-node/src/nakamoto_node/relayer.rs +++ b/testnet/stacks-node/src/nakamoto_node/relayer.rs @@ -250,6 +250,48 @@ impl MinerStopHandle { } } +/// Information necessary to determine when to extend a tenure +pub struct TenureExtendTime { + /// The time at which we determined that we should tenure-extend + time: Instant, + /// The amount of time we should wait before tenure-extending + timeout: Duration, +} + +impl TenureExtendTime { + /// Create a new `TenureExtendTime` with a delayed `timeout` + pub fn delayed(timeout: Duration) -> Self { + Self { + time: Instant::now(), + timeout, + } + } + + /// Create a new `TenureExtendTime` with no `timeout` + pub fn immediate() -> Self { + Self { + time: Instant::now(), + timeout: Duration::from_secs(0), + } + } + + /// Should we attempt to tenure-extend? + pub fn should_extend(&self) -> bool { + // We set the time, but have we waited long enough? + self.time.elapsed() > self.timeout + } + + // Amount of time elapsed since we decided to tenure-extend + pub fn elapsed(&self) -> Duration { + self.time.elapsed() + } + + // The timeout specified when we decided to tenure-extend + pub fn timeout(&self) -> Duration { + self.timeout + } +} + /// Relayer thread /// * accepts network results and stores blocks and microblocks /// * forwards new blocks, microblocks, and transactions to the p2p thread @@ -319,8 +361,8 @@ pub struct RelayerThread { last_committed: Option, /// Timeout for waiting for the first block in a tenure before submitting a block commit new_tenure_timeout: Option, - /// Timeout for waiting for a BlockFound in a subsequent tenure before trying to extend our own - tenure_extend_timeout: Option, + /// Time to wait before attempting a tenure extend + tenure_extend_time: Option, } impl RelayerThread { @@ -380,7 +422,7 @@ impl RelayerThread { next_initiative: Instant::now() + Duration::from_millis(next_initiative_delay), last_committed: None, new_tenure_timeout: None, - tenure_extend_timeout: None, + tenure_extend_time: None, } } @@ -505,7 +547,7 @@ impl RelayerThread { SortitionDB::get_canonical_stacks_chain_tip_hash(self.sortdb.conn()) .expect("FATAL: failed to query sortition DB for stacks tip"); - self.tenure_extend_timeout = None; + self.tenure_extend_time = None; if sn.sortition { // a sortition happened @@ -535,11 +577,18 @@ impl RelayerThread { sn.consensus_hash, mining_pkh_opt, ) { - Ok(Some(_)) => { + Ok(Some((_, wait_for_miner))) => { // we can continue our ongoing tenure, but we should give the new winning miner // a chance to send their BlockFound first. - debug!("Relayer: Did not win sortition, but am mining the ongoing tenure. Allowing the new miner some time to come online before trying to continue."); - self.tenure_extend_timeout = Some(Instant::now()); + if wait_for_miner { + debug!("Relayer: Did not win sortition, but am mining the ongoing tenure. Allowing the new miner some time to come online before trying to continue."); + self.tenure_extend_time = Some(TenureExtendTime::delayed( + self.config.miner.tenure_extend_wait_timeout, + )); + } else { + debug!("Relayer: Did not win sortition, but am mining the ongoing tenure. Will try to continue tenure immediately."); + self.tenure_extend_time = Some(TenureExtendTime::immediate()); + } return Some(MinerDirective::StopTenure); } Ok(None) => { @@ -611,7 +660,7 @@ impl RelayerThread { "Relayer: ongoing tenure {} already represents last-winning snapshot", &stacks_tip_sn.consensus_hash ); - self.tenure_extend_timeout = Some(Instant::now()); + self.tenure_extend_time = Some(TenureExtendTime::immediate()); false } else { // stacks tip's snapshot may be an ancestor of the last-won sortition. @@ -654,7 +703,7 @@ impl RelayerThread { &last_winning_snapshot.consensus_hash ); // prepare to extend after our BlockFound gets mined. - self.tenure_extend_timeout = Some(Instant::now()); + self.tenure_extend_time = Some(TenureExtendTime::immediate()); return Some(MinerDirective::BeginTenure { parent_tenure_start: StacksBlockId( last_winning_snapshot.winning_stacks_block_hash.clone().0, @@ -675,7 +724,9 @@ impl RelayerThread { // by someone else -- there's a chance that this other miner will produce a // BlockFound in the interim. debug!("Relayer: Did not win last winning snapshot despite mining the ongoing tenure, so allowing the new miner some time to come online."); - self.tenure_extend_timeout = Some(Instant::now()); + self.tenure_extend_time = Some(TenureExtendTime::delayed( + self.config.miner.tenure_extend_wait_timeout, + )); return None; } return Some(MinerDirective::ContinueTenure { @@ -1348,10 +1399,10 @@ impl RelayerThread { /// Assumes that the caller has already checked that the given miner has _not_ won the new /// sortition. /// - /// Returns Ok(Some(stacks-tip-election-snapshot)) if the last-winning miner needs to extend. - /// For now, this only happens if the miner's election snapshot was the last-known valid and - /// non-empty snapshot. In the future, this function may return Ok(Some(..)) if the node - /// determines that a subsequent miner won sortition, but never came online. + /// Returns Ok(Some(stacks-tip-election-snapshot, wait-for-miner) if the last-winning miner should attempt to extend + /// This can happen for two seperate reasons: + /// - the miner's election snapshot was the last-known valid and non-empty snapshot and therefore should extend immediately + /// - the node determines that a subsequent miner won sortition, but has not yet produced a valid block and should wait-for-miner before extending /// /// Returns OK(None) if the last-winning miner should not extend its tenure. /// @@ -1361,7 +1412,7 @@ impl RelayerThread { chain_state: &mut StacksChainState, new_burn_view: ConsensusHash, mining_key_opt: Option, - ) -> Result, NakamotoNodeError> { + ) -> Result, NakamotoNodeError> { let Some(mining_pkh) = mining_key_opt else { return Ok(None); }; @@ -1407,23 +1458,25 @@ impl RelayerThread { return Ok(None); } - // For now, only allow the miner to extend its tenure if won the highest valid sortition. - // There cannot be any higher sortitions that are valid (as defined above). - // - // In the future, the miner will be able to extend its tenure even if there are higher - // valid sortitions, but only if it determines that the miners of those sortitions are - // offline. + // Allow the miner to extend its tenure if won the highest valid sortition IFF + // it determines that the miners of the sortition fails to produce a block + // by the required timeout. if let Some(highest_valid_sortition) = Self::find_highest_valid_sortition( sortdb, chain_state, &sort_tip, &canonical_stacks_snapshot.consensus_hash, )? { - info!("Relayer: will not extend tenure -- we won sortition {}, but the highest valid sortition is {}", &canonical_stacks_snapshot.consensus_hash, &highest_valid_sortition.consensus_hash); - return Ok(None); + // TODO: I don't understand why this works? HELP??? + if sort_tip.consensus_hash != highest_valid_sortition.consensus_hash { + info!("Relayer: will not extend tenure -- we won sortition {}, but the highest valid sortition is {}", &canonical_stacks_snapshot.consensus_hash, &highest_valid_sortition.consensus_hash); + return Ok(None); + } + info!("Relayer: MAY extend tenure -- we won sortition {}, but must give miner time to produce a valid block for the highest valid sortition {}", &canonical_stacks_snapshot.consensus_hash, &highest_valid_sortition.consensus_hash); + return Ok(Some((canonical_stacks_snapshot, true))); } - - Ok(Some(canonical_stacks_snapshot)) + // There cannot be any higher sortitions that are valid (as defined above). + Ok(Some((canonical_stacks_snapshot, false))) } /// Attempt to continue a miner's tenure into the next burn block. @@ -1431,7 +1484,8 @@ impl RelayerThread { /// elected the local view of the canonical Stacks fork's ongoing tenure. /// /// This function assumes that the caller has checked that the sortition referred to by - /// `new_burn_view` does not have a sortition winner. + /// `new_burn_view` does not have a sortition winner or that the winner has not produced a + /// valid block yet. fn continue_tenure(&mut self, new_burn_view: ConsensusHash) -> Result<(), NakamotoNodeError> { if let Err(e) = self.stop_tenure() { error!("Relayer: Failed to stop tenure: {e:?}"); @@ -1440,7 +1494,7 @@ impl RelayerThread { debug!("Relayer: successfully stopped tenure; will try to continue."); let mining_pkh_opt = self.get_mining_key_pkh(); - let Some(canonical_stacks_tip_election_snapshot) = Self::can_continue_tenure( + let Some((canonical_stacks_tip_election_snapshot, _)) = Self::can_continue_tenure( &self.sortdb, &mut self.chainstate, new_burn_view.clone(), @@ -1758,38 +1812,30 @@ impl RelayerThread { } /// Try to start up a tenure-extend. - /// Only do this if the miner won the highest valid sortition but the burn view has changed. - /// In the future, the miner will also try to extend its tenure if a subsequent miner appears - /// to be offline. + /// Only do this if: + /// - the miner won the highest valid sortition but the burn view has changed. + /// - the subsequent miner appears to be offline. fn try_continue_tenure(&mut self) { - if self.tenure_extend_timeout.is_none() { - return; - } - - // time to poll to see if we should begin a tenure-extend? - let deadline_passed = self - .tenure_extend_timeout - .map(|tenure_extend_timeout| { - let deadline_passed = - tenure_extend_timeout.elapsed() > self.config.miner.tenure_extend_poll_secs; - if !deadline_passed { - test_debug!( - "Relayer: will not try to tenure-extend yet ({} <= {})", - tenure_extend_timeout.elapsed().as_secs(), - self.config.miner.tenure_extend_poll_secs.as_secs() - ); - } - deadline_passed - }) - .unwrap_or(false); - - if !deadline_passed { + // Should begin a tenure-extend? + if let Some(tenure_extend_time) = &self.tenure_extend_time { + if !tenure_extend_time.should_extend() { + test_debug!( + "Relayer: will not try to tenure-extend yet ({} <= {})", + tenure_extend_time.elapsed().as_secs(), + tenure_extend_time.timeout().as_secs() + ); + return; + } + } else { + // No tenure extend time set, so nothing to do. return; } // reset timer so we can try again if for some reason a miner was already running (e.g. a // blockfound from earlier). - self.tenure_extend_timeout = Some(Instant::now()); + self.tenure_extend_time = Some(TenureExtendTime::delayed( + self.config.miner.tenure_extend_poll_timeout, + )); // try to extend, but only if we aren't already running a thread for the current or newer // burnchain view diff --git a/testnet/stacks-node/src/tests/nakamoto_integrations.rs b/testnet/stacks-node/src/tests/nakamoto_integrations.rs index 4099ce64f2..54bb65d777 100644 --- a/testnet/stacks-node/src/tests/nakamoto_integrations.rs +++ b/testnet/stacks-node/src/tests/nakamoto_integrations.rs @@ -10897,7 +10897,7 @@ fn test_tenure_extend_from_flashblocks() { assert_ne!(sort_tip.consensus_hash, election_tip.consensus_hash); // we can, however, continue the tenure - let canonical_stacks_tip = RelayerThread::can_continue_tenure( + let (canonical_stacks_tip, wait) = RelayerThread::can_continue_tenure( &sortdb, &mut chainstate, sort_tip.consensus_hash.clone(), @@ -10905,6 +10905,7 @@ fn test_tenure_extend_from_flashblocks() { ) .unwrap() .unwrap(); + assert!(!wait); assert_eq!(canonical_stacks_tip, election_tip); // if we didn't win the last block -- tantamount to the sortition winner miner key being diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index d384065cbd..ab9aaf653c 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -12727,3 +12727,883 @@ fn tenure_extend_cost_threshold() { signer_test.shutdown(); } + +/// Test a scenario where: +/// Two miners boot to Nakamoto. +/// Miner 1 wins the first tenure. +/// Miner 1 proposes a block N with a TenureChangeCause::BlockFound +/// Signers accept and the stacks tip advances to N +/// Miner 2 wins the second tenure B. +/// Miner 2 proposes block N+1' AFTER signers' block proposal timeout. +/// Signers reject block N+1' and mark miner 2 as malicious +/// Miner 1 proposes block N+1 with a TenureChangeCause::Extended +/// Signers accept and the stacks tip advances to N+1 +/// Miner 2 wins the third tenure C and proposes a block N+2 with a TenureChangeCause::BlockFound +/// Signers accept block N+2. +/// +/// Asserts: +/// - Block N contains the TenureChangeCause::BlockFound +/// - Block N+1 contains the TenureChangeCause::Extended +/// - Block N+2 contains the TenureChangeCause::BlockFound +/// - The stacks tip advances to N+2 +#[test] +#[ignore] +fn prev_miner_extends_if_incoming_miner_fails_to_mine() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let num_signers = 5; + + let btc_miner_1_seed = vec![1, 1, 1, 1]; + let btc_miner_2_seed = vec![2, 2, 2, 2]; + let btc_miner_1_pk = Keychain::default(btc_miner_1_seed.clone()).get_pub_key(); + let btc_miner_2_pk = Keychain::default(btc_miner_2_seed.clone()).get_pub_key(); + + let node_1_rpc = gen_random_port(); + let node_1_p2p = gen_random_port(); + let node_2_rpc = gen_random_port(); + let node_2_p2p = gen_random_port(); + + debug!("Node 1 bound at (p2p={node_1_p2p}, rpc={node_1_rpc})"); + debug!("Node 2 bound at (p2p={node_2_p2p}, rpc={node_2_rpc})"); + + let localhost = "127.0.0.1"; + let node_1_rpc_bind = format!("{localhost}:{node_1_rpc}"); + let node_2_rpc_bind = format!("{localhost}:{node_2_rpc}"); + let mut node_2_listeners = Vec::new(); + + let max_nakamoto_tenures = 30; + + let block_proposal_timeout = Duration::from_secs(30); + info!("------------------------- Test Setup -------------------------"); + // partition the signer set so that ~half are listening and using node 1 for RPC and events, + // and the rest are using node 2 + + let mut signer_test: SignerTest = SignerTest::new_with_config_modifications( + num_signers, + vec![], + |signer_config| { + let node_host = if signer_config.endpoint.port() % 2 == 0 { + &node_1_rpc_bind + } else { + &node_2_rpc_bind + }; + signer_config.node_host = node_host.to_string(); + signer_config.block_proposal_timeout = block_proposal_timeout; + }, + |config| { + config.miner.tenure_extend_wait_timeout = block_proposal_timeout; + config.node.rpc_bind = format!("{localhost}:{node_1_rpc}"); + config.node.p2p_bind = format!("{localhost}:{node_1_p2p}"); + config.node.data_url = format!("http://{localhost}:{node_1_rpc}"); + config.node.p2p_address = format!("{localhost}:{node_1_p2p}"); + config.miner.wait_on_interim_blocks = Duration::from_secs(5); + config.node.pox_sync_sample_secs = 30; + config.burnchain.pox_reward_length = Some(max_nakamoto_tenures); + + config.node.seed = btc_miner_1_seed.clone(); + config.node.local_peer_seed = btc_miner_1_seed.clone(); + config.burnchain.local_mining_public_key = Some(btc_miner_1_pk.to_hex()); + config.miner.mining_key = Some(Secp256k1PrivateKey::from_seed(&[1])); + + config.events_observers.retain(|listener| { + let Ok(addr) = std::net::SocketAddr::from_str(&listener.endpoint) else { + warn!( + "Cannot parse {} to a socket, assuming it isn't a signer-listener binding", + listener.endpoint + ); + return true; + }; + if addr.port() % 2 == 0 || addr.port() == test_observer::EVENT_OBSERVER_PORT { + return true; + } + node_2_listeners.push(listener.clone()); + false + }) + }, + Some(vec![btc_miner_1_pk, btc_miner_2_pk]), + None, + ); + let conf = signer_test.running_nodes.conf.clone(); + let mut conf_node_2 = conf.clone(); + conf_node_2.node.rpc_bind = format!("{localhost}:{node_2_rpc}"); + conf_node_2.node.p2p_bind = format!("{localhost}:{node_2_p2p}"); + conf_node_2.node.data_url = format!("http://{localhost}:{node_2_rpc}"); + conf_node_2.node.p2p_address = format!("{localhost}:{node_2_p2p}"); + conf_node_2.node.seed = btc_miner_2_seed.clone(); + conf_node_2.burnchain.local_mining_public_key = Some(btc_miner_2_pk.to_hex()); + conf_node_2.node.local_peer_seed = btc_miner_2_seed; + conf_node_2.miner.mining_key = Some(Secp256k1PrivateKey::from_seed(&[2])); + conf_node_2.node.miner = true; + conf_node_2.events_observers.clear(); + conf_node_2.events_observers.extend(node_2_listeners); + assert!(!conf_node_2.events_observers.is_empty()); + + let node_1_sk = Secp256k1PrivateKey::from_seed(&conf.node.local_peer_seed); + let node_1_pk = StacksPublicKey::from_private(&node_1_sk); + + conf_node_2.node.working_dir = format!("{}-1", conf_node_2.node.working_dir); + + conf_node_2.node.set_bootstrap_nodes( + format!("{}@{}", &node_1_pk.to_hex(), conf.node.p2p_bind), + conf.burnchain.chain_id, + conf.burnchain.peer_version, + ); + + let mut run_loop_2 = boot_nakamoto::BootRunLoop::new(conf_node_2.clone()).unwrap(); + let run_loop_stopper_2 = run_loop_2.get_termination_switch(); + let rl2_coord_channels = run_loop_2.coordinator_channels(); + let Counters { + naka_submitted_commits: rl2_commits, + naka_skip_commit_op: rl2_skip_commit_op, + naka_submitted_commit_last_stacks_tip: rl2_commit_last_stacks_tip, + .. + } = run_loop_2.counters(); + + let blocks_mined1 = signer_test.running_nodes.nakamoto_blocks_mined.clone(); + + info!("------------------------- Pause Miner 2's Block Commits -------------------------"); + + // Make sure Miner 2 cannot win a sortition at first. + rl2_skip_commit_op.set(true); + + info!("------------------------- Boot to Epoch 3.0 -------------------------"); + + let run_loop_2_thread = thread::Builder::new() + .name("run_loop_2".into()) + .spawn(move || run_loop_2.start(None, 0)) + .unwrap(); + + signer_test.boot_to_epoch_3(); + + wait_for(120, || { + let Some(node_1_info) = get_chain_info_opt(&conf) else { + return Ok(false); + }; + let Some(node_2_info) = get_chain_info_opt(&conf_node_2) else { + return Ok(false); + }; + Ok(node_1_info.stacks_tip_height == node_2_info.stacks_tip_height) + }) + .expect("Timed out waiting for boostrapped node to catch up to the miner"); + + let mining_pk_1 = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); + let mining_pk_2 = StacksPublicKey::from_private(&conf_node_2.miner.mining_key.unwrap()); + let mining_pkh_1 = Hash160::from_node_public_key(&mining_pk_1); + let mining_pkh_2 = Hash160::from_node_public_key(&mining_pk_2); + debug!("The mining key for miner 1 is {mining_pkh_1}"); + debug!("The mining key for miner 2 is {mining_pkh_2}"); + + info!("------------------------- Reached Epoch 3.0 -------------------------"); + + let burnchain = signer_test.running_nodes.conf.get_burnchain(); + let sortdb = burnchain.open_sortition_db(true).unwrap(); + + let get_burn_height = || { + SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) + .unwrap() + .block_height + }; + let starting_peer_height = get_chain_info(&conf).stacks_tip_height; + let starting_burn_height = get_burn_height(); + let mut btc_blocks_mined = 0; + + info!("------------------------- Pause Miner 1's Block Commit -------------------------"); + // Make sure miner 1 doesn't submit any further block commits for the next tenure BEFORE mining the bitcoin block + signer_test + .running_nodes + .nakamoto_test_skip_commit_op + .set(true); + + info!("------------------------- Miner 1 Mines a Normal Tenure A -------------------------"); + let blocks_processed_before_1 = blocks_mined1.load(Ordering::SeqCst); + let nmb_old_blocks = test_observer::get_blocks().len(); + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + signer_test + .running_nodes + .btc_regtest_controller + .build_next_block(1); + btc_blocks_mined += 1; + + // assure we have a successful sortition that miner A won + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()).unwrap(); + assert!(tip.sortition); + assert_eq!(tip.miner_pk_hash.unwrap(), mining_pkh_1); + + // wait for the new block to be processed + wait_for(60, || { + let stacks_height = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + Ok( + blocks_mined1.load(Ordering::SeqCst) > blocks_processed_before_1 + && stacks_height > stacks_height_before + && test_observer::get_blocks().len() > nmb_old_blocks, + ) + }) + .unwrap(); + + info!( + "------------------------- Verify Tenure Change Tx in Miner 1's Block N -------------------------" + ); + verify_last_block_contains_tenure_change_tx(TenureChangeCause::BlockFound); + + info!("------------------------- Submit Miner 2 Block Commit -------------------------"); + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + let rl2_commits_before = rl2_commits.load(Ordering::SeqCst); + // Unpause miner 2's block commits + rl2_skip_commit_op.set(false); + + // Ensure the miner 2 submits a block commit before mining the bitcoin block + wait_for(30, || { + Ok(rl2_commits.load(Ordering::SeqCst) > rl2_commits_before) + }) + .unwrap(); + + // Make miner 2 also fail to submit any FURTHER block commits + rl2_skip_commit_op.set(true); + + let burn_height_before = get_burn_height(); + // Pause the block proposal broadcast so that miner 2 will be unable to broadcast its + // tenure change proposal BEFORE the block_proposal_timeout and will be marked invalid. + TEST_BROADCAST_STALL.set(true); + + info!("------------------------- Miner 2 Mines an Empty Tenure B -------------------------"; + "burn_height_before" => burn_height_before, + ); + + next_block_and( + &mut signer_test.running_nodes.btc_regtest_controller, + 60, + || Ok(get_burn_height() > burn_height_before), + ) + .unwrap(); + btc_blocks_mined += 1; + + // assure we have a successful sortition that miner 2 won + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()).unwrap(); + assert!(tip.sortition); + assert_eq!(tip.miner_pk_hash.unwrap(), mining_pkh_2); + + // Make sure that miner 2 gets marked invalid by not proposing a block BEFORE block_proposal_timeout + std::thread::sleep(block_proposal_timeout.add(Duration::from_secs(1))); + + let stacks_height_after = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + assert_eq!(stacks_height_before, stacks_height_after); + + let nmb_old_blocks = test_observer::get_blocks().len(); + // Unpause both miner's block proposals + TEST_BROADCAST_STALL.set(false); + + info!("------------------------- Wait for Miner 2's Block N+1' ------------------------"; + "stacks_height_before" => %stacks_height_before, + "nmb_old_blocks" => %nmb_old_blocks); + + // wait for the new block to be processed + wait_for(30, || { + let stacks_height = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + Ok(stacks_height > stacks_height_before + && test_observer::get_blocks().len() > nmb_old_blocks) + }) + .expect("Timed out waiting for block to be mined and processed"); + + info!("------------------------- Verify Miner 2's N+1' was Rejected and Miner 1's N+1 Accepted-------------------------"); + + let mut miner_1_block_n_1 = None; + let mut miner_2_block_n_1 = None; + + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + let SignerMessage::BlockProposal(proposal) = message else { + continue; + }; + let miner_pk = proposal.block.header.recover_miner_pk().unwrap(); + let block_stacks_height = proposal.block.header.chain_length; + if block_stacks_height != stacks_height_before + 1 { + continue; + } + if miner_pk == mining_pk_1 { + miner_1_block_n_1 = Some(proposal.block); + } else if miner_pk == mining_pk_2 { + miner_2_block_n_1 = Some(proposal.block); + } + } + Ok(miner_1_block_n_1.is_some() && miner_2_block_n_1.is_some()) + }) + .expect("Timed out waiting for N+1 and N+1' block proposals from miners 1 and 2"); + + let miner_1_block_n_1 = miner_1_block_n_1.expect("No block proposal from miner 1"); + let miner_2_block_n_1 = miner_2_block_n_1.expect("No block proposal from miner 2"); + + // Miner 2's proposed block should get rejected by all the signers + let mut found_miner_2_rejections = HashSet::new(); + let mut found_miner_1_accepts = HashSet::new(); + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + match message { + SignerMessage::BlockResponse(BlockResponse::Accepted(BlockAccepted { + signer_signature_hash, + signature, + .. + })) => { + if signer_signature_hash == miner_1_block_n_1.header.signer_signature_hash() { + found_miner_1_accepts.insert(signature); + } + } + SignerMessage::BlockResponse(BlockResponse::Rejected(BlockRejection { + signer_signature_hash, + signature, + .. + })) => { + if signer_signature_hash == miner_2_block_n_1.header.signer_signature_hash() { + found_miner_2_rejections.insert(signature); + } + } + _ => {} + } + } + Ok(found_miner_2_rejections.len() >= num_signers * 3 / 10 + && found_miner_1_accepts.len() >= num_signers * 7 / 10) + }) + .expect("Timed out waiting for expeceted block responses"); + + let nakamoto_blocks = test_observer::get_mined_nakamoto_blocks(); + let last_mined = nakamoto_blocks.last().unwrap(); + assert_eq!( + last_mined.signer_signature_hash, + miner_1_block_n_1.header.signer_signature_hash() + ); + let tip_block_header_hash = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip; + assert_eq!(tip_block_header_hash.to_string(), last_mined.block_hash); + + info!( + "------------------------- Verify Tenure Change Extend Tx in Miner 1's Block N+1 -------------------------" + ); + verify_last_block_contains_tenure_change_tx(TenureChangeCause::Extended); + + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + assert_eq!( + get_chain_info(&conf).stacks_tip_height, + stacks_height_before + ); + info!("------------------------- Unpause Miner 2's Block Commits -------------------------"); + let stacks_height_before = get_chain_info(&conf).stacks_tip_height; + let rl2_commits_before = rl2_commits.load(Ordering::SeqCst); + // Unpause miner 2's commits + rl2_skip_commit_op.set(false); + + // Ensure that both miners' commits point at the stacks tip + wait_for(30, || { + let last_committed_2 = rl2_commit_last_stacks_tip.load(Ordering::SeqCst); + Ok(last_committed_2 >= stacks_height_before + && rl2_commits.load(Ordering::SeqCst) > rl2_commits_before) + }) + .expect("Timed out waiting for block commit from Miner 2"); + + let nmb_old_blocks = test_observer::get_blocks().len(); + let burn_height_before = get_burn_height(); + let block_before = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) + .unwrap() + .block_height; + + info!("------------------------- Miner 2 Mines a Normal Tenure C -------------------------"; + "nmb_old_blocks" => %nmb_old_blocks, + "burn_height_before" => burn_height_before); + + next_block_and( + &mut signer_test.running_nodes.btc_regtest_controller, + 60, + || { + Ok(get_burn_height() > burn_height_before + && SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) + .unwrap() + .block_height + > block_before) + }, + ) + .unwrap(); + btc_blocks_mined += 1; + + // assure we have a successful sortition that miner 2 won + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()).unwrap(); + assert!(tip.sortition); + assert_eq!(tip.miner_pk_hash.unwrap(), mining_pkh_2); + + info!("------------------------- Wait for Miner 2's Block N+2 -------------------------"; + "stacks_height_before" => %stacks_height_before, + "nmb_old_blocks" => %nmb_old_blocks); + + wait_for(30, || { + let stacks_height = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + Ok(stacks_height > stacks_height_before) + }) + .expect("Timed out waiting for block N+2 to be mined and processed"); + + info!( + "------------------------- Verify Tenure Change Tx in Miner 2's Block N+2 -------------------------" + ); + verify_last_block_contains_tenure_change_tx(TenureChangeCause::BlockFound); + + info!( + "------------------------- Confirm Burn and Stacks Block Heights -------------------------" + ); + assert_eq!(get_burn_height(), starting_burn_height + btc_blocks_mined); + assert_eq!( + signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height, + starting_peer_height + 3 + ); + + info!("------------------------- Shutdown -------------------------"); + rl2_coord_channels + .lock() + .expect("Mutex poisoned") + .stop_chains_coordinator(); + run_loop_stopper_2.store(false, Ordering::SeqCst); + run_loop_2_thread.join().unwrap(); + signer_test.shutdown(); +} + +/// Test a scenario where: +/// Two miners boot to Nakamoto. +/// Miner 1 wins the first tenure. +/// Miner 1 proposes a block N with a TenureChangeCause::BlockFound +/// Signers accept and the stacks tip advances to N +/// Miner 2 wins the second tenure B. +/// Miner 2 proposes block N+1 with a TenureChangeCause::BlockFound +/// Signers accept and the stacks tip advances to N +/// +/// Asserts: +/// - Block N contains the TenureChangeCause::BlockFound +/// - Block N+1 contains the TenureChangeCause::BlockFound +/// - The stacks tip advances to N+1 +/// - Miner 1 does not produce a tenure extend block +#[test] +#[ignore] +fn prev_miner_will_not_extend_if_incoming_miner_mines() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let num_signers = 5; + + let btc_miner_1_seed = vec![1, 1, 1, 1]; + let btc_miner_2_seed = vec![2, 2, 2, 2]; + let btc_miner_1_pk = Keychain::default(btc_miner_1_seed.clone()).get_pub_key(); + let btc_miner_2_pk = Keychain::default(btc_miner_2_seed.clone()).get_pub_key(); + + let node_1_rpc = gen_random_port(); + let node_1_p2p = gen_random_port(); + let node_2_rpc = gen_random_port(); + let node_2_p2p = gen_random_port(); + + debug!("Node 1 bound at (p2p={node_1_p2p}, rpc={node_1_rpc})"); + debug!("Node 2 bound at (p2p={node_2_p2p}, rpc={node_2_rpc})"); + + let localhost = "127.0.0.1"; + let node_1_rpc_bind = format!("{localhost}:{node_1_rpc}"); + let node_2_rpc_bind = format!("{localhost}:{node_2_rpc}"); + let mut node_2_listeners = Vec::new(); + + let max_nakamoto_tenures = 30; + + let block_proposal_timeout = Duration::from_secs(100); + let tenure_extend_wait_timeout = Duration::from_secs(20); + info!("------------------------- Test Setup -------------------------"); + // partition the signer set so that ~half are listening and using node 1 for RPC and events, + // and the rest are using node 2 + + let mut signer_test: SignerTest = SignerTest::new_with_config_modifications( + num_signers, + vec![], + |signer_config| { + let node_host = if signer_config.endpoint.port() % 2 == 0 { + &node_1_rpc_bind + } else { + &node_2_rpc_bind + }; + signer_config.node_host = node_host.to_string(); + signer_config.block_proposal_timeout = block_proposal_timeout; + }, + |config| { + config.miner.tenure_extend_wait_timeout = tenure_extend_wait_timeout; + config.node.rpc_bind = format!("{localhost}:{node_1_rpc}"); + config.node.p2p_bind = format!("{localhost}:{node_1_p2p}"); + config.node.data_url = format!("http://{localhost}:{node_1_rpc}"); + config.node.p2p_address = format!("{localhost}:{node_1_p2p}"); + config.miner.wait_on_interim_blocks = Duration::from_secs(5); + config.node.pox_sync_sample_secs = 30; + config.burnchain.pox_reward_length = Some(max_nakamoto_tenures); + + config.node.seed = btc_miner_1_seed.clone(); + config.node.local_peer_seed = btc_miner_1_seed.clone(); + config.burnchain.local_mining_public_key = Some(btc_miner_1_pk.to_hex()); + config.miner.mining_key = Some(Secp256k1PrivateKey::from_seed(&[1])); + + config.events_observers.retain(|listener| { + let Ok(addr) = std::net::SocketAddr::from_str(&listener.endpoint) else { + warn!( + "Cannot parse {} to a socket, assuming it isn't a signer-listener binding", + listener.endpoint + ); + return true; + }; + if addr.port() % 2 == 0 || addr.port() == test_observer::EVENT_OBSERVER_PORT { + return true; + } + node_2_listeners.push(listener.clone()); + false + }) + }, + Some(vec![btc_miner_1_pk, btc_miner_2_pk]), + None, + ); + let conf = signer_test.running_nodes.conf.clone(); + let mut conf_node_2 = conf.clone(); + conf_node_2.node.rpc_bind = format!("{localhost}:{node_2_rpc}"); + conf_node_2.node.p2p_bind = format!("{localhost}:{node_2_p2p}"); + conf_node_2.node.data_url = format!("http://{localhost}:{node_2_rpc}"); + conf_node_2.node.p2p_address = format!("{localhost}:{node_2_p2p}"); + conf_node_2.node.seed = btc_miner_2_seed.clone(); + conf_node_2.burnchain.local_mining_public_key = Some(btc_miner_2_pk.to_hex()); + conf_node_2.node.local_peer_seed = btc_miner_2_seed; + conf_node_2.miner.mining_key = Some(Secp256k1PrivateKey::from_seed(&[2])); + conf_node_2.node.miner = true; + conf_node_2.events_observers.clear(); + conf_node_2.events_observers.extend(node_2_listeners); + assert!(!conf_node_2.events_observers.is_empty()); + + let node_1_sk = Secp256k1PrivateKey::from_seed(&conf.node.local_peer_seed); + let node_1_pk = StacksPublicKey::from_private(&node_1_sk); + + conf_node_2.node.working_dir = format!("{}-1", conf_node_2.node.working_dir); + + conf_node_2.node.set_bootstrap_nodes( + format!("{}@{}", &node_1_pk.to_hex(), conf.node.p2p_bind), + conf.burnchain.chain_id, + conf.burnchain.peer_version, + ); + + let mut run_loop_2 = boot_nakamoto::BootRunLoop::new(conf_node_2.clone()).unwrap(); + let run_loop_stopper_2 = run_loop_2.get_termination_switch(); + let rl2_coord_channels = run_loop_2.coordinator_channels(); + let Counters { + naka_submitted_commits: rl2_commits, + naka_skip_commit_op: rl2_skip_commit_op, + .. + } = run_loop_2.counters(); + + let blocks_mined1 = signer_test.running_nodes.nakamoto_blocks_mined.clone(); + + info!("------------------------- Pause Miner 2's Block Commits -------------------------"); + + // Make sure Miner 2 cannot win a sortition at first. + rl2_skip_commit_op.set(true); + + info!("------------------------- Boot to Epoch 3.0 -------------------------"); + + let run_loop_2_thread = thread::Builder::new() + .name("run_loop_2".into()) + .spawn(move || run_loop_2.start(None, 0)) + .unwrap(); + + signer_test.boot_to_epoch_3(); + + wait_for(120, || { + let Some(node_1_info) = get_chain_info_opt(&conf) else { + return Ok(false); + }; + let Some(node_2_info) = get_chain_info_opt(&conf_node_2) else { + return Ok(false); + }; + Ok(node_1_info.stacks_tip_height == node_2_info.stacks_tip_height) + }) + .expect("Timed out waiting for boostrapped node to catch up to the miner"); + + let mining_pk_1 = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); + let mining_pk_2 = StacksPublicKey::from_private(&conf_node_2.miner.mining_key.unwrap()); + let mining_pkh_1 = Hash160::from_node_public_key(&mining_pk_1); + let mining_pkh_2 = Hash160::from_node_public_key(&mining_pk_2); + debug!("The mining key for miner 1 is {mining_pkh_1}"); + debug!("The mining key for miner 2 is {mining_pkh_2}"); + + info!("------------------------- Reached Epoch 3.0 -------------------------"); + + let burnchain = signer_test.running_nodes.conf.get_burnchain(); + let sortdb = burnchain.open_sortition_db(true).unwrap(); + + let get_burn_height = || { + SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) + .unwrap() + .block_height + }; + let starting_peer_height = get_chain_info(&conf).stacks_tip_height; + let starting_burn_height = get_burn_height(); + let mut btc_blocks_mined = 0; + + info!("------------------------- Pause Miner 1's Block Commit -------------------------"); + // Make sure miner 1 doesn't submit any further block commits for the next tenure BEFORE mining the bitcoin block + signer_test + .running_nodes + .nakamoto_test_skip_commit_op + .set(true); + + info!("------------------------- Miner 1 Mines a Normal Tenure A -------------------------"); + let blocks_processed_before_1 = blocks_mined1.load(Ordering::SeqCst); + let nmb_old_blocks = test_observer::get_blocks().len(); + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + signer_test + .running_nodes + .btc_regtest_controller + .build_next_block(1); + btc_blocks_mined += 1; + + // assure we have a successful sortition that miner A won + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()).unwrap(); + assert!(tip.sortition); + assert_eq!(tip.miner_pk_hash.unwrap(), mining_pkh_1); + + // wait for the new block to be processed + wait_for(60, || { + let stacks_height = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + Ok( + blocks_mined1.load(Ordering::SeqCst) > blocks_processed_before_1 + && stacks_height > stacks_height_before + && test_observer::get_blocks().len() > nmb_old_blocks, + ) + }) + .unwrap(); + + info!( + "------------------------- Verify Tenure Change Tx in Miner 1's Block N -------------------------" + ); + verify_last_block_contains_tenure_change_tx(TenureChangeCause::BlockFound); + + info!("------------------------- Submit Miner 2 Block Commit -------------------------"); + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + let rl2_commits_before = rl2_commits.load(Ordering::SeqCst); + // Unpause miner 2's block commits + rl2_skip_commit_op.set(false); + + // Ensure the miner 2 submits a block commit before mining the bitcoin block + wait_for(30, || { + Ok(rl2_commits.load(Ordering::SeqCst) > rl2_commits_before) + }) + .unwrap(); + + // Make miner 2 also fail to submit any FURTHER block commits + rl2_skip_commit_op.set(true); + + let burn_height_before = get_burn_height(); + + info!("------------------------- Miner 2 Mines Tenure B -------------------------"; + "burn_height_before" => burn_height_before, + "stacks_height_before" => stacks_height_before + ); + + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + next_block_and( + &mut signer_test.running_nodes.btc_regtest_controller, + 60, + || { + Ok(get_burn_height() > burn_height_before + && signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height + > stacks_height_before) + }, + ) + .unwrap(); + btc_blocks_mined += 1; + + // assure we have a successful sortition that miner 2 won + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()).unwrap(); + assert!(tip.sortition); + assert_eq!(tip.miner_pk_hash.unwrap(), mining_pkh_2); + + info!("------------------------- Get Miner 2's N+1' block -------------------------"); + + let mut miner_2_block_n_1 = None; + + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + let SignerMessage::BlockProposal(proposal) = message else { + continue; + }; + let miner_pk = proposal.block.header.recover_miner_pk().unwrap(); + let block_stacks_height = proposal.block.header.chain_length; + if block_stacks_height != stacks_height_before + 1 { + continue; + } + assert_eq!(miner_pk, mining_pk_2); + miner_2_block_n_1 = Some(proposal.block); + return Ok(true); + } + Ok(false) + }) + .expect("Timed out waiting for N+1 from miner 2"); + + let mut miner_2_block_n_1 = miner_2_block_n_1.expect("No block proposal from miner 2"); + + // Miner 2's proposed block should get approved and pushed + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + if let SignerMessage::BlockPushed(pushed_block) = message { + if pushed_block.header.signer_signature_hash() + == miner_2_block_n_1.header.signer_signature_hash() + { + miner_2_block_n_1 = pushed_block; + return Ok(true); + } + } + } + Ok(false) + }) + .expect("Timed out waiting for expeceted block responses"); + + let tip_block_header_hash = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip; + assert_eq!( + tip_block_header_hash.to_string(), + miner_2_block_n_1.header.block_hash().to_string() + ); + + info!( + "------------------------- Verify Tenure Change Block Found Tx in Miner 2's Block N+1 -------------------------" + ); + assert_eq!( + miner_2_block_n_1 + .get_tenure_change_tx_payload() + .unwrap() + .cause, + TenureChangeCause::BlockFound + ); + + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + + info!("------------------------- Ensure Miner 1 Never Isues a Tenure Extend -------------------------"; + "stacks_height_before" => %stacks_height_before, + "nmb_old_blocks" => %nmb_old_blocks); + + // Ensure the tenure extend wait timeout is passed + std::thread::sleep(tenure_extend_wait_timeout.add(Duration::from_secs(1))); + + assert!(wait_for(30, || { + let stacks_height = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + Ok(stacks_height > stacks_height_before) + }) + .is_err()); + + info!( + "------------------------- Confirm Burn and Stacks Block Heights -------------------------" + ); + assert_eq!(get_burn_height(), starting_burn_height + btc_blocks_mined); + assert_eq!( + signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height, + starting_peer_height + 2 + ); + + info!("------------------------- Shutdown -------------------------"); + rl2_coord_channels + .lock() + .expect("Mutex poisoned") + .stop_chains_coordinator(); + run_loop_stopper_2.store(false, Ordering::SeqCst); + run_loop_2_thread.join().unwrap(); + signer_test.shutdown(); +}