Use Reconfigurator simulator in planner integration tests (#9473)

This is a followon to #9456 and finishes converting the rest of the
planner tests to use the simulator.

I think this is a pretty solid win overall; there's a lot less
boilerplate (e.g., in just raw LoC we dropped several hundred in the
integration test module). I also picked up some small places where our
simulator was missing bits we needed from integration tests. The biggest
boilerplate reduction is that running the planner changes from this:

```rust
    let blueprint = Planner::new_based_on(
        logctx.log.clone(),
        &parent,
        &input,
        "test_blueprint",
        &collection,
        PlannerRng::from_seed((TEST_NAME, "blueprint")),
    )
    .expect("failed to create planner")
    .plan()
    .expect("failed to plan");
```

to this:

```rust
    let blueprint = sim.run_planner().expect("planning succeeded");
```

because `sim` internally is able to construct all the other args needed
for planning. It always uses the latest blueprint and latest inventory
collection, which has been fine in practice. (If we needed to run the
planner with different ones for some reason, it'd be easy to add a
`run_planner_with_...`-style method to specify different ones.)

There are still some rough edges; a couple off the top of my head:

* A `SystemDescription` has a set of `active` and `not_yet` Nexus zones,
but those don't influence the planning runs. Instead, we have to use the
`set_explicit_{active,not_yet}_nexus_zones()` methods on
`SimConfigBuilder`. I haven't dug into this.
* Inventory changes from outside the blueprint are kind of a pain, so
the zone safety checks that rely on status from cockroach, NTP, and
internal DNS have a bunch of `sim.inventory_edit_latest_low_level(...)`
calls to make up various statuses. This still feels pretty boilerplate-y
but doesn't seem any _worse_ than it was before. I don't know if it
makes sense to grow the simulator to contain this kind of status.

Author: jgallagher

Cargo.lock

@@ -88,7 +88,7 @@ mod log_capture;
 pub mod test_utils;
 
 /// REPL state
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 struct ReconfiguratorSim {
     // The simulator currently being used.
     sim: Simulator,

dev-tools/reconfigurator-cli/src/lib.rs

@@ -7,19 +7,30 @@
 
 use crate::ReconfiguratorSim;
 use anyhow::Context;
+use nexus_inventory::CollectionBuilder;
+use nexus_inventory::now_db_precision;
 use nexus_reconfigurator_blippy::Blippy;
 use nexus_reconfigurator_blippy::BlippyReportSortKey;
+use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
 use nexus_reconfigurator_planning::example::ExampleSystemBuilder;
 use nexus_reconfigurator_planning::system::SledBuilder;
+use nexus_reconfigurator_planning::system::SystemDescription;
 use nexus_reconfigurator_simulation::BlueprintId;
 use nexus_reconfigurator_simulation::CollectionId;
+use nexus_reconfigurator_simulation::SimState;
 use nexus_reconfigurator_simulation::SimStateBuilder;
 use nexus_reconfigurator_simulation::errors::KeyError;
 use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::BlueprintSource;
+use nexus_types::deployment::ClickhousePolicy;
 use nexus_types::deployment::PlanningInput;
+use nexus_types::external_api::views::SledPolicy;
+use nexus_types::inventory::Collection;
 use omicron_uuid_kinds::SledUuid;
 use slog::Logger;
+use std::sync::Arc;
 
+#[derive(Debug, Clone)]
 pub struct ReconfiguratorCliTestState {
     sim: ReconfiguratorSim,
 }
@@ -53,11 +64,21 @@ impl ReconfiguratorCliTestState {
     }
 
     /// Get the specified blueprint.
-    pub fn blueprint(&self, id: BlueprintId) -> Result<&Blueprint, KeyError> {
+    pub fn blueprint(
+        &self,
+        id: BlueprintId,
+    ) -> Result<&Arc<Blueprint>, KeyError> {
         let state = self.sim.current_state();
         state.system().resolve_and_get_blueprint(id)
     }
 
+    /// Get the specified inventory collection.
+    pub fn inventory(&self, id: CollectionId) -> Result<&Collection, KeyError> {
+        let system = self.current_state().system();
+        let id = system.resolve_collection_id(id)?;
+        system.get_collection(&id)
+    }
+
     /// Run the blueprint planner, using the latest blueprint and inventory
     /// collection as input.
     ///
@@ -74,7 +95,7 @@ impl ReconfiguratorCliTestState {
     /// "blippy clean" blueprints, and since this is used throughout the
     /// planner's tests, we want to assert that any blueprint we plan is indeed
     /// blippy clean.
-    pub fn run_planner(&mut self) -> anyhow::Result<Blueprint> {
+    pub fn run_planner(&mut self) -> anyhow::Result<Arc<Blueprint>> {
         let output =
             self.sim.run_planner(BlueprintId::Latest, CollectionId::Latest)?;
         println!("{output}");
@@ -103,7 +124,7 @@ impl ReconfiguratorCliTestState {
     ///
     /// Panics if the latest blueprint and current planning input have any
     /// blippy notes.
-    pub fn assert_latest_blueprint_is_blippy_clean(&self) -> Blueprint {
+    pub fn assert_latest_blueprint_is_blippy_clean(&self) -> Arc<Blueprint> {
         let blueprint = self
             .blueprint(BlueprintId::Latest)
             .expect("always have a latest blueprint");
@@ -120,7 +141,17 @@ impl ReconfiguratorCliTestState {
             panic!("expected blippy report for blueprint to have no notes");
         }
 
-        blueprint.clone()
+        Arc::clone(blueprint)
+    }
+
+    /// Read the current internal simulator state.
+    pub fn current_state(&self) -> &SimState {
+        self.sim.current_state()
+    }
+
+    /// Read the current system description.
+    pub fn current_description(&self) -> &SystemDescription {
+        self.current_state().system().description()
     }
 
     /// Change the internal simulator state.
@@ -138,6 +169,20 @@ impl ReconfiguratorCliTestState {
         Ok(ret)
     }
 
+    /// State change helper: change only the system description.
+    pub fn change_description<F, T>(
+        &mut self,
+        description: &str,
+        f: F,
+    ) -> anyhow::Result<T>
+    where
+        F: FnOnce(&mut SystemDescription) -> anyhow::Result<T>,
+    {
+        self.change_state(description, |state| {
+            f(state.system_mut().description_mut())
+        })
+    }
+
     /// State change helper: generate a new inventory collection from the
     /// current simulator state.
     pub fn generate_inventory(
@@ -151,15 +196,139 @@ impl ReconfiguratorCliTestState {
         })
     }
 
+    /// State change helper: generate a new inventory collection from the
+    /// current simulator state.
+    ///
+    /// `f` provides an opportunity to customize the collection.
+    pub fn generate_inventory_customized<F, T>(
+        &mut self,
+        description: &str,
+        f: F,
+    ) -> anyhow::Result<T>
+    where
+        F: FnOnce(&mut CollectionBuilder) -> anyhow::Result<T>,
+    {
+        self.change_state(description, |state| {
+            let mut builder = state.to_collection_builder()?;
+            let result = f(&mut builder)?;
+            state.system_mut().add_collection(builder.build())?;
+            Ok(result)
+        })
+    }
+
+    /// State change helper: create a new latest inventory collection, then pass
+    /// it to `f` which is allowed to edit it in abnormal ways that cannot be
+    /// done via `CollectionBuilder`.
+    pub fn inventory_edit_latest_raw<F, T>(
+        &mut self,
+        description: &str,
+        f: F,
+    ) -> anyhow::Result<T>
+    where
+        F: FnOnce(&mut Collection) -> anyhow::Result<T>,
+    {
+        self.change_state(description, |state| {
+            let mut collection = state.to_collection_builder()?.build();
+            let result = f(&mut collection)?;
+            state.system_mut().add_collection(collection)?;
+            Ok(result)
+        })
+    }
+
+    /// State change helper: set the Clickhouse cluster policy.
+    pub fn set_clickhouse_policy(
+        &mut self,
+        description: &str,
+        policy: ClickhousePolicy,
+    ) {
+        self.change_description(description, |desc| {
+            desc.clickhouse_policy(policy);
+            Ok(())
+        })
+        .expect("closure can't fail");
+    }
+
+    /// State change helper: edit the latest blueprint, inserting a new latest
+    /// blueprint.
+    pub fn blueprint_edit_latest<F>(
+        &mut self,
+        description: &str,
+        f: F,
+    ) -> anyhow::Result<Arc<Blueprint>>
+    where
+        F: FnOnce(&mut BlueprintBuilder<'_>) -> anyhow::Result<()>,
+    {
+        let log = self.sim.log.clone();
+        self.change_state(description, |state| {
+            let rng = state.rng_mut().next_planner_rng();
+            let system = state.system_mut();
+            let blueprint = system
+                .get_blueprint(
+                    &system.resolve_blueprint_id(BlueprintId::Latest),
+                )
+                .expect("always have a latest blueprint");
+            let mut builder = BlueprintBuilder::new_based_on(
+                &log,
+                blueprint,
+                "ReconfiguratorCliTestState",
+                rng,
+            )?;
+            f(&mut builder)?;
+            let blueprint = Arc::new(builder.build(BlueprintSource::Test));
+            system.add_blueprint(Arc::clone(&blueprint))?;
+            Ok(blueprint)
+        })
+    }
+
+    /// State change helper: create a new latest blueprint that is a clone of
+    /// the current latest blueprint (but with a new ID and correct parent ID),
+    /// then pass it to `f` which is allowed to edit it in abnormal ways that
+    /// cannot be done via `BlueprintBuilder`.
+    pub fn blueprint_edit_latest_raw<F>(
+        &mut self,
+        description: &str,
+        f: F,
+    ) -> anyhow::Result<Arc<Blueprint>>
+    where
+        F: FnOnce(&mut Blueprint) -> anyhow::Result<()>,
+    {
+        self.change_state(description, |state| {
+            let mut rng = state.rng_mut().next_planner_rng();
+            let system = state.system_mut();
+            let parent_blueprint = system
+                .get_blueprint(
+                    &system.resolve_blueprint_id(BlueprintId::Latest),
+                )
+                .expect("always have a latest blueprint");
+            let mut blueprint = Arc::clone(parent_blueprint);
+
+            {
+                let blueprint = Arc::make_mut(&mut blueprint);
+
+                // Update metadata fields to make this a new blueprint.
+                blueprint.id = rng.next_blueprint();
+                blueprint.parent_blueprint_id = Some(parent_blueprint.id);
+                blueprint.time_created = now_db_precision();
+
+                // Perform whatever modifications the caller wants.
+                f(blueprint)?;
+            }
+
+            system.add_blueprint(Arc::clone(&blueprint))?;
+
+            Ok(blueprint)
+        })
+    }
+
     /// State change helper: add a new sled, returning its ID.
-    pub fn add_sled(&mut self, description: &str) -> anyhow::Result<SledUuid> {
-        self.add_sled_customized(description, |sled| sled)
+    pub fn sled_add(&mut self, description: &str) -> anyhow::Result<SledUuid> {
+        self.sled_add_customized(description, |sled| sled)
     }
 
     /// State change helper: add a new sled, returning its ID.
     ///
     /// `f` provides an opportunity to customize the sled.
-    pub fn add_sled_customized<F>(
+    pub fn sled_add_customized<F>(
         &mut self,
         description: &str,
         f: F,
@@ -175,6 +344,21 @@ impl ReconfiguratorCliTestState {
         })
     }
 
+    /// State change helper: expunge a sled.
+    pub fn sled_expunge(
+        &mut self,
+        description: &str,
+        sled_id: SledUuid,
+    ) -> anyhow::Result<()> {
+        self.change_state(description, |state| {
+            state
+                .system_mut()
+                .description_mut()
+                .sled_set_policy(sled_id, SledPolicy::Expunged)?;
+            Ok(())
+        })
+    }
+
     /// State change helper: for each active sled described by `blueprint`,
     /// update the simulate sled's config to match.
     ///

dev-tools/reconfigurator-cli/src/test_utils.rs

@@ -2253,10 +2253,13 @@ fn check_planning_input_network_records_appear_in_blueprint(
                 }
             }
             _ => {
-                blippy.push_planning_input_note(
-                    Severity::Fatal,
-                    PlanningInputKind::NicWithUnknownOpteSubnet(nic_entry),
-                );
+                // Ignore localhost (used by the test suite).
+                if !nic_entry.nic.ip.is_loopback() {
+                    blippy.push_planning_input_note(
+                        Severity::Fatal,
+                        PlanningInputKind::NicWithUnknownOpteSubnet(nic_entry),
+                    );
+                }
             }
         }
     }

nexus/reconfigurator/blippy/src/checks.rs

@@ -55,13 +55,15 @@ uuid.workspace = true
 omicron-workspace-hack.workspace = true
 
 [dev-dependencies]
+assert_matches.workspace = true
 dns-server.workspace = true
 dropshot.workspace = true
 expectorate.workspace = true
 hex-literal.workspace = true
 hickory-resolver.workspace = true
 internal-dns-types.workspace = true
 maplit.workspace = true
+nexus-reconfigurator-simulation.workspace = true
 omicron-common = { workspace = true, features = ["testing"] }
 omicron-test-utils.workspace = true
 proptest.workspace = true

nexus/reconfigurator/planning/Cargo.toml

@@ -919,10 +919,7 @@ impl<'a> BlueprintBuilder<'a> {
     }
 
     /// Expunge everything on a sled.
-    pub(crate) fn expunge_sled(
-        &mut self,
-        sled_id: SledUuid,
-    ) -> Result<(), Error> {
+    pub fn expunge_sled(&mut self, sled_id: SledUuid) -> Result<(), Error> {
         let editor = self.sled_editors.get_mut(&sled_id).ok_or_else(|| {
             Error::Planner(anyhow!("tried to expunge unknown sled {sled_id}"))
         })?;