From 2a4d461627e966d73736984de4f852baadaf41a9 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 6 Jan 2025 18:13:21 +0300 Subject: [PATCH 01/11] previously, interactions plans were comprised of a flat sequence of operations that did not reflect the internal structure, in which they were actually concatenations of properties, which are a coherent set of interactions that are meaningful by themselves. this commit introduces this semantic layer into the data model by turning interaction plans into a sequence of properties, which are a sequence of interactions --- simulator/generation/plan.rs | 124 +++++++++++++++++++++++------------ simulator/main.rs | 11 +++- 2 files changed, 91 insertions(+), 44 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index dbf6dd7f60..fafd3741bb 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -19,20 +19,53 @@ use super::{pick, pick_index}; pub(crate) type ResultSet = Result>>; pub(crate) struct InteractionPlan { - pub(crate) plan: Vec, + pub(crate) plan: Vec, pub(crate) stack: Vec, pub(crate) interaction_pointer: usize, + pub(crate) secondary_pointer: usize, +} + +pub(crate) struct Property { + pub(crate) name: Option, + pub(crate) interactions: Vec, +} + +impl Property { + pub(crate) fn new(name: Option, interactions: Vec) -> Self { + Self { name, interactions } + } + + pub(crate) fn anonymous(interactions: Vec) -> Self { + Self { + name: None, + interactions, + } + } } impl Display for InteractionPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for interaction in &self.plan { - match interaction { - Interaction::Query(query) => writeln!(f, "{};", query)?, - Interaction::Assertion(assertion) => { - writeln!(f, "-- ASSERT: {};", assertion.message)? + for property in &self.plan { + if let Some(name) = &property.name { + writeln!(f, "-- begin testing '{}'", name)?; + } + + for interaction in &property.interactions { + if property.name.is_some() { + write!(f, "\t")?; } - Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + + match interaction { + Interaction::Query(query) => writeln!(f, "{};", query)?, + Interaction::Assertion(assertion) => { + writeln!(f, "-- ASSERT: {};", assertion.message)? + } + Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + } + } + + if let Some(name) = &property.name { + writeln!(f, "-- end testing '{}'", name)?; } } @@ -93,11 +126,9 @@ impl Display for Fault { } } -pub(crate) struct Interactions(Vec); - -impl Interactions { +impl Property { pub(crate) fn shadow(&self, env: &mut SimulatorEnv) { - for interaction in &self.0 { + for interaction in &self.interactions { match interaction { Interaction::Query(query) => match query { Query::Create(create) => { @@ -129,29 +160,28 @@ impl InteractionPlan { plan: Vec::new(), stack: Vec::new(), interaction_pointer: 0, + secondary_pointer: 0, } } - pub(crate) fn push(&mut self, interaction: Interaction) { - self.plan.push(interaction); - } - pub(crate) fn stats(&self) -> InteractionStats { let mut read = 0; let mut write = 0; let mut delete = 0; let mut create = 0; - for interaction in &self.plan { - match interaction { - Interaction::Query(query) => match query { - Query::Select(_) => read += 1, - Query::Insert(_) => write += 1, - Query::Delete(_) => delete += 1, - Query::Create(_) => create += 1, - }, - Interaction::Assertion(_) => {} - Interaction::Fault(_) => {} + for property in &self.plan { + for interaction in &property.interactions { + match interaction { + Interaction::Query(query) => match query { + Query::Select(_) => read += 1, + Query::Insert(_) => write += 1, + Query::Delete(_) => delete += 1, + Query::Create(_) => create += 1, + }, + Interaction::Assertion(_) => {} + Interaction::Fault(_) => {} + } } } @@ -182,7 +212,11 @@ impl ArbitraryFrom for InteractionPlan { // First create at least one table let create_query = Create::arbitrary(rng); env.tables.push(create_query.table.clone()); - plan.push(Interaction::Query(Query::Create(create_query))); + + plan.plan.push(Property { + name: Some("initial table creation".to_string()), + interactions: vec![Interaction::Query(Query::Create(create_query))], + }); while plan.plan.len() < num_interactions { log::debug!( @@ -190,10 +224,10 @@ impl ArbitraryFrom for InteractionPlan { plan.plan.len(), num_interactions ); - let interactions = Interactions::arbitrary_from(rng, &(&env, plan.stats())); - interactions.shadow(&mut env); + let property = Property::arbitrary_from(rng, &(&env, plan.stats())); + property.shadow(&mut env); - plan.plan.extend(interactions.0.into_iter()); + plan.plan.push(property); } log::info!("Generated plan with {} interactions", plan.plan.len()); @@ -306,7 +340,7 @@ impl Interaction { } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { // Get a random table let table = pick(&env.tables, rng); // Pick a random column @@ -354,10 +388,13 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Inte }), }); - Interactions(vec![insert_query, select_query, assertion]) + Property::new( + Some("select contains inserted value".to_string()), + vec![insert_query, select_query, assertion], + ) } -fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Property { let create_query = Create::arbitrary(rng); let table_name = create_query.table.name.clone(); let cq1 = Interaction::Query(Query::Create(create_query.clone())); @@ -378,31 +415,34 @@ fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv }), }); - Interactions(vec![cq1, cq2, assertion]) + Property::new( + Some("creating the same table twice fails".to_string()), + vec![cq1, cq2, assertion], + ) } -fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Property { let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng))); - Interactions(vec![create_query]) + Property::anonymous(vec![create_query]) } -fn random_read(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn random_read(rng: &mut R, env: &SimulatorEnv) -> Property { let select_query = Interaction::Query(Query::Select(Select::arbitrary_from(rng, &env.tables))); - Interactions(vec![select_query]) + Property::anonymous(vec![select_query]) } -fn random_write(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn random_write(rng: &mut R, env: &SimulatorEnv) -> Property { let table = pick(&env.tables, rng); let insert_query = Interaction::Query(Query::Insert(Insert::arbitrary_from(rng, table))); - Interactions(vec![insert_query]) + Property::anonymous(vec![insert_query]) } -fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Property { let fault = Interaction::Fault(Fault::Disconnect); - Interactions(vec![fault]) + Property::anonymous(vec![fault]) } -impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { +impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { fn arbitrary_from( rng: &mut R, (env, stats): &(&SimulatorEnv, InteractionStats), diff --git a/simulator/main.rs b/simulator/main.rs index 52c33d5ec7..794c9c0414 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -266,7 +266,7 @@ fn execute_plan( return Ok(()); } - let interaction = &plan.plan[plan.interaction_pointer]; + let interaction = &plan.plan[plan.interaction_pointer].interactions[plan.secondary_pointer]; if let SimConnection::Disconnected = connection { log::info!("connecting {}", connection_index); @@ -275,7 +275,14 @@ fn execute_plan( match execute_interaction(env, connection_index, interaction, &mut plan.stack) { Ok(_) => { log::debug!("connection {} processed", connection_index); - plan.interaction_pointer += 1; + if plan.secondary_pointer + 1 + >= plan.plan[plan.interaction_pointer].interactions.len() + { + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } else { + plan.secondary_pointer += 1; + } } Err(err) => { log::error!("error {}", err); From daa77feea13feaf2c48622d4650da9f6af8b18d7 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 6 Jan 2025 19:04:29 +0300 Subject: [PATCH 02/11] add assumptions to the interactions, where a failing assumption stops the execution of the current property and switches to the next one. --- simulator/generation/plan.rs | 72 ++++++++++++++++++++++++++++++++---- simulator/main.rs | 54 +++++++++++++++++++++------ 2 files changed, 108 insertions(+), 18 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index fafd3741bb..ca3a6d5036 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -57,6 +57,9 @@ impl Display for InteractionPlan { match interaction { Interaction::Query(query) => writeln!(f, "{};", query)?, + Interaction::Assumption(assumption) => { + writeln!(f, "-- ASSUME: {};", assumption.message)? + } Interaction::Assertion(assertion) => { writeln!(f, "-- ASSERT: {};", assertion.message)? } @@ -93,6 +96,7 @@ impl Display for InteractionStats { pub(crate) enum Interaction { Query(Query), + Assumption(Assertion), Assertion(Assertion), Fault(Fault), } @@ -101,13 +105,14 @@ impl Display for Interaction { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Query(query) => write!(f, "{}", query), + Self::Assumption(assumption) => write!(f, "ASSUME: {}", assumption.message), Self::Assertion(assertion) => write!(f, "ASSERT: {}", assertion.message), Self::Fault(fault) => write!(f, "FAULT: {}", fault), } } } -type AssertionFunc = dyn Fn(&Vec) -> bool; +type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> bool; pub(crate) struct Assertion { pub(crate) func: Box, @@ -148,6 +153,7 @@ impl Property { Query::Select(_) => {} }, Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} Interaction::Fault(_) => {} } } @@ -180,6 +186,7 @@ impl InteractionPlan { Query::Create(_) => create += 1, }, Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} Interaction::Fault(_) => {} } } @@ -285,31 +292,67 @@ impl Interaction { Self::Assertion(_) => { unreachable!("unexpected: this function should only be called on queries") } - Interaction::Fault(_) => { + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on queries") + } + Self::Fault(_) => { unreachable!("unexpected: this function should only be called on queries") } } } - pub(crate) fn execute_assertion(&self, stack: &Vec) -> Result<()> { + pub(crate) fn execute_assertion( + &self, + stack: &Vec, + env: &SimulatorEnv, + ) -> Result<()> { match self { Self::Query(_) => { unreachable!("unexpected: this function should only be called on assertions") } Self::Assertion(assertion) => { - if !assertion.func.as_ref()(stack) { + if !assertion.func.as_ref()(stack, env) { return Err(limbo_core::LimboError::InternalError( assertion.message.clone(), )); } Ok(()) } + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on assertions") + } Self::Fault(_) => { unreachable!("unexpected: this function should only be called on assertions") } } } + pub(crate) fn execute_assumption( + &self, + stack: &Vec, + env: &SimulatorEnv, + ) -> Result<()> { + match self { + Self::Query(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + Self::Assertion(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + Self::Assumption(assumption) => { + if !assumption.func.as_ref()(stack, env) { + return Err(limbo_core::LimboError::InternalError( + assumption.message.clone(), + )); + } + Ok(()) + } + Self::Fault(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + } + } + pub(crate) fn execute_fault(&self, env: &mut SimulatorEnv, conn_index: usize) -> Result<()> { match self { Self::Query(_) => { @@ -318,6 +361,9 @@ impl Interaction { Self::Assertion(_) => { unreachable!("unexpected: this function should only be called on faults") } + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on faults") + } Self::Fault(fault) => { match fault { Fault::Disconnect => { @@ -358,6 +404,18 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop row.push(value); } } + + // Check that the table exists + let assumption = Interaction::Assumption(Assertion { + message: format!("table {} exists", table.name), + func: Box::new({ + let table_name = table.name.clone(); + move |_: &Vec, env: &SimulatorEnv| { + env.tables.iter().any(|t| t.name == table_name) + } + }), + }); + // Insert the row let insert_query = Interaction::Query(Query::Insert(Insert { table: table.name.clone(), @@ -379,7 +437,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop column.name, value, ), - func: Box::new(move |stack: &Vec| { + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let rows = stack.last().unwrap(); match rows { Ok(rows) => rows.iter().any(|r| r == &row), @@ -390,7 +448,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop Property::new( Some("select contains inserted value".to_string()), - vec![insert_query, select_query, assertion], + vec![assumption, insert_query, select_query, assertion], ) } @@ -404,7 +462,7 @@ fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv message: "creating two tables with the name should result in a failure for the second query" .to_string(), - func: Box::new(move |stack: &Vec| { + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let last = stack.last().unwrap(); match last { Ok(_) => false, diff --git a/simulator/main.rs b/simulator/main.rs index 794c9c0414..ec65e228f4 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -273,15 +273,27 @@ fn execute_plan( env.connections[connection_index] = SimConnection::Connected(env.db.connect()); } else { match execute_interaction(env, connection_index, interaction, &mut plan.stack) { - Ok(_) => { + Ok(next_execution) => { log::debug!("connection {} processed", connection_index); - if plan.secondary_pointer + 1 - >= plan.plan[plan.interaction_pointer].interactions.len() - { - plan.interaction_pointer += 1; - plan.secondary_pointer = 0; - } else { - plan.secondary_pointer += 1; + // Move to the next interaction or property + match next_execution { + ExecutionContinuation::NextInteraction => { + if plan.secondary_pointer + 1 + >= plan.plan[plan.interaction_pointer].interactions.len() + { + // If we have reached the end of the interactions for this property, move to the next property + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } else { + // Otherwise, move to the next interaction + plan.secondary_pointer += 1; + } + } + ExecutionContinuation::NextProperty => { + // Skip to the next property + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } } } Err(err) => { @@ -294,12 +306,23 @@ fn execute_plan( Ok(()) } +/// The next point of control flow after executing an interaction. +/// `execute_interaction` uses this type in conjunction with a result, where +/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case +/// indicates the next step in the plan. +enum ExecutionContinuation { + /// Default continuation, execute the next interaction. + NextInteraction, + /// Typically used in the case of preconditions failures, skip to the next property. + NextProperty, +} + fn execute_interaction( env: &mut SimulatorEnv, connection_index: usize, interaction: &Interaction, stack: &mut Vec, -) -> Result<()> { +) -> Result { log::info!("executing: {}", interaction); match interaction { generation::plan::Interaction::Query(_) => { @@ -314,15 +337,24 @@ fn execute_interaction( stack.push(results); } generation::plan::Interaction::Assertion(_) => { - interaction.execute_assertion(stack)?; + interaction.execute_assertion(stack, env)?; stack.clear(); } + generation::plan::Interaction::Assumption(_) => { + let assumption_result = interaction.execute_assumption(stack, env); + stack.clear(); + + if assumption_result.is_err() { + log::warn!("assumption failed: {:?}", assumption_result); + return Ok(ExecutionContinuation::NextProperty); + } + } Interaction::Fault(_) => { interaction.execute_fault(env, connection_index)?; } } - Ok(()) + Ok(ExecutionContinuation::NextInteraction) } fn compare_equal_rows(a: &[Vec], b: &[Vec]) { From cc56276c3ab92aed808e4f0f05315c10f3e599d9 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Tue, 7 Jan 2025 13:40:29 +0300 Subject: [PATCH 03/11] add execution history to the simulator, the history records three indexes(connection, interaction pointer, and secondary pointer) that can uniquely identify the executed interaction at any point. we will use the history for shrinking purposes. --- simulator/generation/plan.rs | 21 ++---- simulator/main.rs | 129 ++++++++++++++++++++++++++--------- 2 files changed, 104 insertions(+), 46 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index ca3a6d5036..0a18d38217 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -1,8 +1,6 @@ use std::{fmt::Display, rc::Rc}; use limbo_core::{Connection, Result, StepResult}; -use rand::SeedableRng; -use rand_chacha::ChaCha8Rng; use crate::{ model::{ @@ -201,19 +199,12 @@ impl InteractionPlan { } } -impl ArbitraryFrom for InteractionPlan { - fn arbitrary_from(rng: &mut R, env: &SimulatorEnv) -> Self { +impl InteractionPlan { + // todo: This is a hack to get around the fact that `ArbitraryFrom` can't take a mutable + // reference of T, so instead write a bespoke function without using the trait system. + pub(crate) fn arbitrary_from(rng: &mut R, env: &mut SimulatorEnv) -> Self { let mut plan = InteractionPlan::new(); - let mut env = SimulatorEnv { - opts: env.opts.clone(), - tables: vec![], - connections: vec![], - io: env.io.clone(), - db: env.db.clone(), - rng: ChaCha8Rng::seed_from_u64(rng.next_u64()), - }; - let num_interactions = env.opts.max_interactions; // First create at least one table @@ -231,8 +222,8 @@ impl ArbitraryFrom for InteractionPlan { plan.plan.len(), num_interactions ); - let property = Property::arbitrary_from(rng, &(&env, plan.stats())); - property.shadow(&mut env); + let property = Property::arbitrary_from(rng, &(env, plan.stats())); + property.shadow(env); plan.plan.push(property); } diff --git a/simulator/main.rs b/simulator/main.rs index ec65e228f4..5491c5780a 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,8 +1,9 @@ #![allow(clippy::arc_with_non_send_sync, dead_code)] use clap::Parser; +use core::panic; +use generation::pick_index; use generation::plan::{Interaction, InteractionPlan, ResultSet}; -use generation::{pick_index, ArbitraryFrom}; -use limbo_core::{Database, Result}; +use limbo_core::{Database, LimboError, Result}; use model::table::Value; use rand::prelude::*; use rand_chacha::ChaCha8Rng; @@ -36,10 +37,12 @@ fn main() { let db_path = output_dir.join("simulator.db"); let plan_path = output_dir.join("simulator.plan"); + let history_path = output_dir.join("simulator.history"); // Print the seed, the locations of the database and the plan file log::info!("database path: {:?}", db_path); log::info!("simulator plan path: {:?}", plan_path); + log::info!("simulator history path: {:?}", history_path); log::info!("seed: {}", seed); std::panic::set_hook(Box::new(move |info| { @@ -73,28 +76,34 @@ fn main() { std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path)); match (result, result2) { - (Ok(Ok(_)), Err(_)) => { + (Ok(ExecutionResult { error: None, .. }), Err(_)) => { log::error!("doublecheck failed! first run succeeded, but second run panicked."); } - (Ok(Err(_)), Err(_)) => { + (Ok(ExecutionResult { error: Some(_), .. }), Err(_)) => { log::error!( "doublecheck failed! first run failed assertion, but second run panicked." ); } - (Err(_), Ok(Ok(_))) => { + (Err(_), Ok(ExecutionResult { error: None, .. })) => { log::error!("doublecheck failed! first run panicked, but second run succeeded."); } - (Err(_), Ok(Err(_))) => { + (Err(_), Ok(ExecutionResult { error: Some(_), .. })) => { log::error!( "doublecheck failed! first run panicked, but second run failed assertion." ); } - (Ok(Ok(_)), Ok(Err(_))) => { + ( + Ok(ExecutionResult { error: None, .. }), + Ok(ExecutionResult { error: Some(_), .. }), + ) => { log::error!( "doublecheck failed! first run succeeded, but second run failed assertion." ); } - (Ok(Err(_)), Ok(Ok(_))) => { + ( + Ok(ExecutionResult { error: Some(_), .. }), + Ok(ExecutionResult { error: None, .. }), + ) => { log::error!( "doublecheck failed! first run failed assertion, but second run succeeded." ); @@ -122,18 +131,32 @@ fn main() { std::fs::rename(&old_db_path, &db_path).unwrap(); std::fs::rename(&old_plan_path, &plan_path).unwrap(); } else if let Ok(result) = result { - match result { - Ok(_) => { + // No panic occurred, so write the history to a file + let f = std::fs::File::create(&history_path).unwrap(); + let mut f = std::io::BufWriter::new(f); + for execution in result.history.history.iter() { + writeln!( + f, + "{} {} {}", + execution.connection_index, execution.interaction_index, execution.secondary_index + ) + .unwrap(); + } + + match result.error { + None => { log::info!("simulation completed successfully"); } - Err(e) => { + Some(e) => { log::error!("simulation failed: {:?}", e); } } } + // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. println!("database path: {:?}", db_path); println!("simulator plan path: {:?}", plan_path); + println!("simulator history path: {:?}", history_path); println!("seed: {}", seed); } @@ -142,7 +165,7 @@ fn run_simulation( cli_opts: &SimulatorCLI, db_path: &Path, plan_path: &Path, -) -> Result<()> { +) -> ExecutionResult { let mut rng = ChaCha8Rng::seed_from_u64(seed); let (create_percent, read_percent, write_percent, delete_percent) = { @@ -160,21 +183,15 @@ fn run_simulation( }; if cli_opts.minimum_size < 1 { - return Err(limbo_core::LimboError::InternalError( - "minimum size must be at least 1".to_string(), - )); + panic!("minimum size must be at least 1"); } if cli_opts.maximum_size < 1 { - return Err(limbo_core::LimboError::InternalError( - "maximum size must be at least 1".to_string(), - )); + panic!("maximum size must be at least 1"); } if cli_opts.maximum_size < cli_opts.minimum_size { - return Err(limbo_core::LimboError::InternalError( - "maximum size must be greater than or equal to minimum size".to_string(), - )); + panic!("maximum size must be greater than or equal to minimum size"); } let opts = SimulatorOpts { @@ -212,7 +229,7 @@ fn run_simulation( log::info!("Generating database interaction plan..."); let mut plans = (1..=env.opts.max_connections) - .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &env)) + .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &mut env)) .collect::>(); let mut f = std::fs::File::create(plan_path).unwrap(); @@ -224,9 +241,6 @@ fn run_simulation( log::info!("Executing database interaction plan..."); let result = execute_plans(&mut env, &mut plans); - if result.is_err() { - log::error!("error executing plans: {:?}", result.as_ref().err()); - } env.io.print_stats(); @@ -235,23 +249,76 @@ fn run_simulation( result } -fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> Result<()> { +struct Execution { + connection_index: usize, + interaction_index: usize, + secondary_index: usize, +} + +impl Execution { + fn new(connection_index: usize, interaction_index: usize, secondary_index: usize) -> Self { + Self { + connection_index, + interaction_index, + secondary_index, + } + } +} + +struct ExecutionHistory { + history: Vec, +} + +impl ExecutionHistory { + fn new() -> Self { + Self { + history: Vec::new(), + } + } +} + +struct ExecutionResult { + history: ExecutionHistory, + error: Option, +} + +impl ExecutionResult { + fn new(history: ExecutionHistory, error: Option) -> Self { + Self { history, error } + } +} + +fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> ExecutionResult { + let mut history = ExecutionHistory::new(); let now = std::time::Instant::now(); // todo: add history here by recording which interaction was executed at which tick for _tick in 0..env.opts.ticks { // Pick the connection to interact with let connection_index = pick_index(env.connections.len(), &mut env.rng); + history.history.push(Execution::new( + connection_index, + plans[connection_index].interaction_pointer, + plans[connection_index].secondary_pointer, + )); // Execute the interaction for the selected connection - execute_plan(env, connection_index, plans)?; + match execute_plan(env, connection_index, plans) { + Ok(_) => {} + Err(err) => { + return ExecutionResult::new(history, Some(err)); + } + } // Check if the maximum time for the simulation has been reached if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { - return Err(limbo_core::LimboError::InternalError( - "maximum time for simulation reached".into(), - )); + return ExecutionResult::new( + history, + Some(limbo_core::LimboError::InternalError( + "maximum time for simulation reached".into(), + )), + ); } } - Ok(()) + ExecutionResult::new(history, None) } fn execute_plan( From 191b586f05737b4b26b71f706255c6f7beac8d7a Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sat, 11 Jan 2025 02:20:22 +0300 Subject: [PATCH 04/11] this commit restructures the interaction generation in order to have better counterexample minimization. - it separates interaction plans from their state of execution - it removes closures from the property definitions, encoding properties as an enum variant, and deriving the closures from the variants. - it adds some naive counterexample minimization capabilities to the Limbo simulator and reduces the plan sizes considerably. - it makes small changes to various points of the simulator for better error reporting, enhancing code readability, small fixes to handle previously missed cases --- simulator/generation/mod.rs | 3 +- simulator/generation/plan.rs | 465 ++++++++++++++--------------- simulator/generation/property.rs | 234 +++++++++++++++ simulator/generation/query.rs | 22 +- simulator/main.rs | 498 +++++++++++++++---------------- simulator/model/query.rs | 20 +- simulator/runner/cli.rs | 6 + simulator/runner/execution.rs | 202 +++++++++++++ simulator/runner/mod.rs | 1 + simulator/shrink/mod.rs | 1 + simulator/shrink/plan.rs | 28 ++ 11 files changed, 979 insertions(+), 501 deletions(-) create mode 100644 simulator/generation/property.rs create mode 100644 simulator/runner/execution.rs create mode 100644 simulator/shrink/mod.rs create mode 100644 simulator/shrink/plan.rs diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 8158b2d171..6107124f00 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -4,6 +4,7 @@ use anarchist_readable_name_generator_lib::readable_name_custom; use rand::{distributions::uniform::SampleUniform, Rng}; pub mod plan; +pub mod property; pub mod query; pub mod table; @@ -21,7 +22,7 @@ pub(crate) fn frequency< R: rand::Rng, N: Sum + PartialOrd + Copy + Default + SampleUniform + SubAssign, >( - choices: Vec<(N, Box T + 'a>)>, + choices: Vec<(N, Box T + 'a>)>, rng: &mut R, ) -> T { let total = choices.iter().map(|(weight, _)| *weight).sum::(); diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 0a18d38217..e286bb34ab 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -1,10 +1,10 @@ -use std::{fmt::Display, rc::Rc}; +use std::{fmt::Display, rc::Rc, vec}; use limbo_core::{Connection, Result, StepResult}; use crate::{ model::{ - query::{Create, Insert, Predicate, Query, Select}, + query::{Create, Insert, Query, Select}, table::Value, }, SimConnection, SimulatorEnv, @@ -12,61 +12,115 @@ use crate::{ use crate::generation::{frequency, Arbitrary, ArbitraryFrom}; -use super::{pick, pick_index}; +use super::{pick, property::Property}; pub(crate) type ResultSet = Result>>; +#[derive(Clone)] pub(crate) struct InteractionPlan { - pub(crate) plan: Vec, + pub(crate) plan: Vec, +} + +pub(crate) struct InteractionPlanState { pub(crate) stack: Vec, pub(crate) interaction_pointer: usize, pub(crate) secondary_pointer: usize, } -pub(crate) struct Property { - pub(crate) name: Option, - pub(crate) interactions: Vec, +#[derive(Clone)] +pub(crate) enum Interactions { + Property(Property), + Query(Query), + Fault(Fault), } -impl Property { - pub(crate) fn new(name: Option, interactions: Vec) -> Self { - Self { name, interactions } +impl Interactions { + pub(crate) fn name(&self) -> Option { + match self { + Interactions::Property(property) => Some(property.name()), + Interactions::Query(_) => None, + Interactions::Fault(_) => None, + } } - pub(crate) fn anonymous(interactions: Vec) -> Self { - Self { - name: None, - interactions, + pub(crate) fn interactions(&self) -> Vec { + match self { + Interactions::Property(property) => property.interactions(), + Interactions::Query(query) => vec![Interaction::Query(query.clone())], + Interactions::Fault(fault) => vec![Interaction::Fault(fault.clone())], } } } -impl Display for InteractionPlan { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for property in &self.plan { - if let Some(name) = &property.name { - writeln!(f, "-- begin testing '{}'", name)?; +impl Interactions { + pub(crate) fn dependencies(&self) -> Vec { + match self { + Interactions::Property(property) => { + property + .interactions() + .iter() + .fold(vec![], |mut acc, i| match i { + Interaction::Query(q) => { + acc.extend(q.dependencies()); + acc + } + _ => acc, + }) } + Interactions::Query(query) => query.dependencies(), + Interactions::Fault(_) => vec![], + } + } - for interaction in &property.interactions { - if property.name.is_some() { - write!(f, "\t")?; - } + pub(crate) fn uses(&self) -> Vec { + match self { + Interactions::Property(property) => { + property + .interactions() + .iter() + .fold(vec![], |mut acc, i| match i { + Interaction::Query(q) => { + acc.extend(q.uses()); + acc + } + _ => acc, + }) + } + Interactions::Query(query) => query.uses(), + Interactions::Fault(_) => vec![], + } + } +} - match interaction { - Interaction::Query(query) => writeln!(f, "{};", query)?, - Interaction::Assumption(assumption) => { - writeln!(f, "-- ASSUME: {};", assumption.message)? - } - Interaction::Assertion(assertion) => { - writeln!(f, "-- ASSERT: {};", assertion.message)? +impl Display for InteractionPlan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for interactions in &self.plan { + match interactions { + Interactions::Property(property) => { + let name = property.name(); + writeln!(f, "-- begin testing '{}'", name)?; + for interaction in property.interactions() { + write!(f, "\t")?; + + match interaction { + Interaction::Query(query) => writeln!(f, "{};", query)?, + Interaction::Assumption(assumption) => { + writeln!(f, "-- ASSUME: {};", assumption.message)? + } + Interaction::Assertion(assertion) => { + writeln!(f, "-- ASSERT: {};", assertion.message)? + } + Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + } } - Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + writeln!(f, "-- end testing '{}'", name)?; + } + Interactions::Fault(fault) => { + writeln!(f, "-- FAULT '{}'", fault)?; + } + Interactions::Query(query) => { + writeln!(f, "{};", query)?; } - } - - if let Some(name) = &property.name { - writeln!(f, "-- end testing '{}'", name)?; } } @@ -74,7 +128,7 @@ impl Display for InteractionPlan { } } -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub(crate) struct InteractionStats { pub(crate) read_count: usize, pub(crate) write_count: usize, @@ -112,11 +166,16 @@ impl Display for Interaction { type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> bool; +enum AssertionAST { + Pick(), +} + pub(crate) struct Assertion { pub(crate) func: Box, pub(crate) message: String, } +#[derive(Debug, Clone)] pub(crate) enum Fault { Disconnect, } @@ -129,43 +188,60 @@ impl Display for Fault { } } -impl Property { +impl Interactions { pub(crate) fn shadow(&self, env: &mut SimulatorEnv) { - for interaction in &self.interactions { - match interaction { - Interaction::Query(query) => match query { - Query::Create(create) => { - if !env.tables.iter().any(|t| t.name == create.table.name) { - env.tables.push(create.table.clone()); - } - } - Query::Insert(insert) => { - let table = env - .tables - .iter_mut() - .find(|t| t.name == insert.table) - .unwrap(); - table.rows.extend(insert.values.clone()); + match self { + Interactions::Property(property) => { + for interaction in property.interactions() { + match interaction { + Interaction::Query(query) => match query { + Query::Create(create) => { + if !env.tables.iter().any(|t| t.name == create.table.name) { + env.tables.push(create.table.clone()); + } + } + Query::Insert(insert) => { + let table = env + .tables + .iter_mut() + .find(|t| t.name == insert.table) + .unwrap(); + table.rows.extend(insert.values.clone()); + } + Query::Delete(_) => todo!(), + Query::Select(_) => {} + }, + Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} + Interaction::Fault(_) => {} } - Query::Delete(_) => todo!(), - Query::Select(_) => {} - }, - Interaction::Assertion(_) => {} - Interaction::Assumption(_) => {} - Interaction::Fault(_) => {} + } } + Interactions::Query(query) => match query { + Query::Create(create) => { + if !env.tables.iter().any(|t| t.name == create.table.name) { + env.tables.push(create.table.clone()); + } + } + Query::Insert(insert) => { + let table = env + .tables + .iter_mut() + .find(|t| t.name == insert.table) + .unwrap(); + table.rows.extend(insert.values.clone()); + } + Query::Delete(_) => todo!(), + Query::Select(_) => {} + }, + Interactions::Fault(_) => {} } } } impl InteractionPlan { pub(crate) fn new() -> Self { - Self { - plan: Vec::new(), - stack: Vec::new(), - interaction_pointer: 0, - secondary_pointer: 0, - } + Self { plan: Vec::new() } } pub(crate) fn stats(&self) -> InteractionStats { @@ -174,19 +250,27 @@ impl InteractionPlan { let mut delete = 0; let mut create = 0; - for property in &self.plan { - for interaction in &property.interactions { - match interaction { - Interaction::Query(query) => match query { - Query::Select(_) => read += 1, - Query::Insert(_) => write += 1, - Query::Delete(_) => delete += 1, - Query::Create(_) => create += 1, - }, - Interaction::Assertion(_) => {} - Interaction::Assumption(_) => {} - Interaction::Fault(_) => {} + for interactions in &self.plan { + match interactions { + Interactions::Property(property) => { + for interaction in &property.interactions() { + if let Interaction::Query(query) = interaction { + match query { + Query::Select(_) => read += 1, + Query::Insert(_) => write += 1, + Query::Delete(_) => delete += 1, + Query::Create(_) => create += 1, + } + } + } } + Interactions::Query(query) => match query { + Query::Select(_) => read += 1, + Query::Insert(_) => write += 1, + Query::Delete(_) => delete += 1, + Query::Create(_) => create += 1, + }, + Interactions::Fault(_) => {} } } @@ -211,10 +295,8 @@ impl InteractionPlan { let create_query = Create::arbitrary(rng); env.tables.push(create_query.table.clone()); - plan.plan.push(Property { - name: Some("initial table creation".to_string()), - interactions: vec![Interaction::Query(Query::Create(create_query))], - }); + plan.plan + .push(Interactions::Query(Query::Create(create_query))); while plan.plan.len() < num_interactions { log::debug!( @@ -222,10 +304,10 @@ impl InteractionPlan { plan.plan.len(), num_interactions ); - let property = Property::arbitrary_from(rng, &(env, plan.stats())); - property.shadow(env); + let interactions = Interactions::arbitrary_from(rng, &(env, plan.stats())); + interactions.shadow(env); - plan.plan.push(property); + plan.plan.push(interactions); } log::info!("Generated plan with {} interactions", plan.plan.len()); @@ -235,60 +317,51 @@ impl InteractionPlan { impl Interaction { pub(crate) fn execute_query(&self, conn: &mut Rc) -> ResultSet { - match self { - Self::Query(query) => { - let query_str = query.to_string(); - let rows = conn.query(&query_str); - if rows.is_err() { - let err = rows.err(); - log::error!( - "Error running query '{}': {:?}", - &query_str[0..query_str.len().min(4096)], - err - ); - return Err(err.unwrap()); - } - let rows = rows.unwrap(); - assert!(rows.is_some()); - let mut rows = rows.unwrap(); - let mut out = Vec::new(); - while let Ok(row) = rows.next_row() { - match row { - StepResult::Row(row) => { - let mut r = Vec::new(); - for el in &row.values { - let v = match el { - limbo_core::Value::Null => Value::Null, - limbo_core::Value::Integer(i) => Value::Integer(*i), - limbo_core::Value::Float(f) => Value::Float(*f), - limbo_core::Value::Text(t) => Value::Text(t.to_string()), - limbo_core::Value::Blob(b) => Value::Blob(b.to_vec()), - }; - r.push(v); - } - - out.push(r); - } - StepResult::IO => {} - StepResult::Interrupt => {} - StepResult::Done => { - break; + if let Self::Query(query) = self { + let query_str = query.to_string(); + let rows = conn.query(&query_str); + if rows.is_err() { + let err = rows.err(); + log::error!( + "Error running query '{}': {:?}", + &query_str[0..query_str.len().min(4096)], + err + ); + return Err(err.unwrap()); + } + let rows = rows.unwrap(); + assert!(rows.is_some()); + let mut rows = rows.unwrap(); + let mut out = Vec::new(); + while let Ok(row) = rows.next_row() { + match row { + StepResult::Row(row) => { + let mut r = Vec::new(); + for el in &row.values { + let v = match el { + limbo_core::Value::Null => Value::Null, + limbo_core::Value::Integer(i) => Value::Integer(*i), + limbo_core::Value::Float(f) => Value::Float(*f), + limbo_core::Value::Text(t) => Value::Text(t.to_string()), + limbo_core::Value::Blob(b) => Value::Blob(b.to_vec()), + }; + r.push(v); } - StepResult::Busy => {} + + out.push(r); + } + StepResult::IO => {} + StepResult::Interrupt => {} + StepResult::Done => { + break; } + StepResult::Busy => {} } - - Ok(out) - } - Self::Assertion(_) => { - unreachable!("unexpected: this function should only be called on queries") - } - Self::Assumption(_) => { - unreachable!("unexpected: this function should only be called on queries") - } - Self::Fault(_) => { - unreachable!("unexpected: this function should only be called on queries") } + + Ok(out) + } else { + unreachable!("unexpected: this function should only be called on queries") } } @@ -377,121 +450,25 @@ impl Interaction { } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { - // Get a random table - let table = pick(&env.tables, rng); - // Pick a random column - let column_index = pick_index(table.columns.len(), rng); - let column = &table.columns[column_index].clone(); - // Generate a random value of the column type - let value = Value::arbitrary_from(rng, &column.column_type); - // Create a whole new row - let mut row = Vec::new(); - for (i, column) in table.columns.iter().enumerate() { - if i == column_index { - row.push(value.clone()); - } else { - let value = Value::arbitrary_from(rng, &column.column_type); - row.push(value); - } - } - - // Check that the table exists - let assumption = Interaction::Assumption(Assertion { - message: format!("table {} exists", table.name), - func: Box::new({ - let table_name = table.name.clone(); - move |_: &Vec, env: &SimulatorEnv| { - env.tables.iter().any(|t| t.name == table_name) - } - }), - }); - - // Insert the row - let insert_query = Interaction::Query(Query::Insert(Insert { - table: table.name.clone(), - values: vec![row.clone()], - })); - - // Select the row - let select_query = Interaction::Query(Query::Select(Select { - table: table.name.clone(), - predicate: Predicate::Eq(column.name.clone(), value.clone()), - })); - - // Check that the row is there - let assertion = Interaction::Assertion(Assertion { - message: format!( - "row [{:?}] not found in table {} after inserting ({} = {})", - row.iter().map(|v| v.to_string()).collect::>(), - table.name, - column.name, - value, - ), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { - let rows = stack.last().unwrap(); - match rows { - Ok(rows) => rows.iter().any(|r| r == &row), - Err(_) => false, - } - }), - }); - - Property::new( - Some("select contains inserted value".to_string()), - vec![assumption, insert_query, select_query, assertion], - ) -} - -fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Property { - let create_query = Create::arbitrary(rng); - let table_name = create_query.table.name.clone(); - let cq1 = Interaction::Query(Query::Create(create_query.clone())); - let cq2 = Interaction::Query(Query::Create(create_query.clone())); - - let assertion = Interaction::Assertion(Assertion { - message: - "creating two tables with the name should result in a failure for the second query" - .to_string(), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { - let last = stack.last().unwrap(); - match last { - Ok(_) => false, - Err(e) => e - .to_string() - .contains(&format!("Table {table_name} already exists")), - } - }), - }); - - Property::new( - Some("creating the same table twice fails".to_string()), - vec![cq1, cq2, assertion], - ) -} - -fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Property { - let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng))); - Property::anonymous(vec![create_query]) +fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Interactions { + Interactions::Query(Query::Create(Create::arbitrary(rng))) } -fn random_read(rng: &mut R, env: &SimulatorEnv) -> Property { - let select_query = Interaction::Query(Query::Select(Select::arbitrary_from(rng, &env.tables))); - Property::anonymous(vec![select_query]) +fn random_read(rng: &mut R, env: &SimulatorEnv) -> Interactions { + Interactions::Query(Query::Select(Select::arbitrary_from(rng, &env.tables))) } -fn random_write(rng: &mut R, env: &SimulatorEnv) -> Property { +fn random_write(rng: &mut R, env: &SimulatorEnv) -> Interactions { let table = pick(&env.tables, rng); - let insert_query = Interaction::Query(Query::Insert(Insert::arbitrary_from(rng, table))); - Property::anonymous(vec![insert_query]) + let insert_query = Query::Insert(Insert::arbitrary_from(rng, table)); + Interactions::Query(insert_query) } -fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Property { - let fault = Interaction::Fault(Fault::Disconnect); - Property::anonymous(vec![fault]) +fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions { + Interactions::Fault(Fault::Disconnect) } -impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { +impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { fn arbitrary_from( rng: &mut R, (env, stats): &(&SimulatorEnv, InteractionStats), @@ -510,8 +487,10 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { frequency( vec![ ( - f64::min(remaining_read, remaining_write), - Box::new(|rng: &mut R| property_insert_select(rng, env)), + f64::min(remaining_read, remaining_write) + remaining_create, + Box::new(|rng: &mut R| { + Interactions::Property(Property::arbitrary_from(rng, &(env, stats))) + }), ), ( remaining_read, @@ -526,10 +505,6 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { Box::new(|rng: &mut R| create_table(rng, env)), ), (1.0, Box::new(|rng: &mut R| random_fault(rng, env))), - ( - remaining_create / 2.0, - Box::new(|rng: &mut R| property_double_create_failure(rng, env)), - ), ], rng, ) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs new file mode 100644 index 0000000000..39956210be --- /dev/null +++ b/simulator/generation/property.rs @@ -0,0 +1,234 @@ +use crate::{ + model::{ + query::{Create, Insert, Predicate, Query, Select}, + table::Value, + }, + runner::env::SimulatorEnv, +}; + +use super::{ + frequency, pick, pick_index, + plan::{Assertion, Interaction, InteractionStats, ResultSet}, + ArbitraryFrom, +}; + +/// Properties are representations of executable specifications +/// about the database behavior. +#[derive(Clone)] +pub(crate) enum Property { + /// Insert-Select is a property in which the inserted row + /// must be in the resulting rows of a select query that has a + /// where clause that matches the inserted row. + /// The execution of the property is as follows + /// INSERT INTO VALUES (...) + /// I_0 + /// I_1 + /// ... + /// I_n + /// SELECT * FROM WHERE + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - The inserted row will not be deleted. + /// - The inserted row will not be updated. + /// - The table `t` will not be renamed, dropped, or altered. + InsertSelect { + /// The insert query + insert: Insert, + /// Additional interactions in the middle of the property + interactions: Vec, + /// The select query + select: Select, + }, + /// Double Create Failure is a property in which creating + /// the same table twice leads to an error. + /// The execution of the property is as follows + /// CREATE TABLE (...) + /// I_0 + /// I_1 + /// ... + /// I_n + /// CREATE TABLE (...) -> Error + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - Table `t` will not be renamed or dropped. + DoubleCreateFailure { + /// The create query + create: Create, + /// Additional interactions in the middle of the property + interactions: Vec, + }, +} + +impl Property { + pub(crate) fn name(&self) -> String { + match self { + Property::InsertSelect { .. } => "Insert-Select".to_string(), + Property::DoubleCreateFailure { .. } => "Double-Create-Failure".to_string(), + } + } + pub(crate) fn interactions(&self) -> Vec { + match self { + Property::InsertSelect { + insert, + interactions: _, // todo: add extensional interactions + select, + } => { + // Check that the row is there + let row = insert + .values + .first() // `.first` is safe, because we know we are inserting a row in the insert select property + .expect("insert query should have at least 1 value") + .clone(); + + let assumption = Interaction::Assumption(Assertion { + message: format!("table {} exists", insert.table), + func: Box::new({ + let table_name = insert.table.clone(); + move |_: &Vec, env: &SimulatorEnv| { + env.tables.iter().any(|t| t.name == table_name) + } + }), + }); + + let assertion = Interaction::Assertion(Assertion { + message: format!( + // todo: add the part inserting ({} = {})", + "row [{:?}] not found in table {}", + row.iter().map(|v| v.to_string()).collect::>(), + insert.table, + ), + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + let rows = stack.last().unwrap(); + match rows { + Ok(rows) => rows.iter().any(|r| r == &row), + Err(_) => false, + } + }), + }); + + vec![ + assumption, + Interaction::Query(Query::Insert(insert.clone())), + Interaction::Query(Query::Select(select.clone())), + assertion, + ] + } + Property::DoubleCreateFailure { + create, + interactions: _, // todo: add extensional interactions + } => { + let table_name = create.table.name.clone(); + let cq1 = Interaction::Query(Query::Create(create.clone())); + let cq2 = Interaction::Query(Query::Create(create.clone())); + + let assertion = Interaction::Assertion(Assertion { + message: + "creating two tables with the name should result in a failure for the second query" + .to_string(), + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + let last = stack.last().unwrap(); + match last { + Ok(_) => false, + Err(e) => e + .to_string() + .contains(&format!("Table {table_name} already exists")), + } + }), + }); + + vec![cq1, cq2, assertion] + } + } + } +} + +fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> (f64, f64, f64) { + let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) + - (stats.read_count as f64)) + .max(0.0); + let remaining_write = ((env.opts.max_interactions as f64 * env.opts.write_percent / 100.0) + - (stats.write_count as f64)) + .max(0.0); + let remaining_create = ((env.opts.max_interactions as f64 * env.opts.create_percent / 100.0) + - (stats.create_count as f64)) + .max(0.0); + + (remaining_read, remaining_write, remaining_create) +} + +fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { + // Get a random table + let table = pick(&env.tables, rng); + // Pick a random column + let column_index = pick_index(table.columns.len(), rng); + let column = &table.columns[column_index].clone(); + // Generate a random value of the column type + let value = Value::arbitrary_from(rng, &column.column_type); + // Create a whole new row + let mut row = Vec::new(); + for (i, column) in table.columns.iter().enumerate() { + if i == column_index { + row.push(value.clone()); + } else { + let value = Value::arbitrary_from(rng, &column.column_type); + row.push(value); + } + } + + // Insert the row + let insert_query = Insert { + table: table.name.clone(), + values: vec![row.clone()], + }; + + // Select the row + let select_query = Select { + table: table.name.clone(), + predicate: Predicate::arbitrary_from( + rng, + &(table, &Predicate::Eq(column.name.clone(), value.clone())), + ), + }; + + Property::InsertSelect { + insert: insert_query, + interactions: Vec::new(), + select: select_query, + } +} + +fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) -> Property { + // Get a random table + let table = pick(&env.tables, rng); + // Create the table + let create_query = Create { + table: table.clone(), + }; + + Property::DoubleCreateFailure { + create: create_query, + interactions: Vec::new(), + } +} + +impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { + fn arbitrary_from( + rng: &mut R, + (env, stats): &(&SimulatorEnv, &InteractionStats), + ) -> Self { + let (remaining_read, remaining_write, remaining_create) = remaining(env, stats); + frequency( + vec![ + ( + f64::min(remaining_read, remaining_write), + Box::new(|rng: &mut R| property_insert_select(rng, env)), + ), + ( + remaining_create / 2.0, + Box::new(|rng: &mut R| property_double_create_failure(rng, env)), + ), + ], + rng, + ) + } +} diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index b39ef6785e..c99638f6dc 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -161,13 +161,13 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { // An AND for false requires at least one of its children to be false if *predicate_value { Predicate::And( - (0..rng.gen_range(1..=3)) + (0..rng.gen_range(0..=3)) .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, true)).0) .collect(), ) } else { // Create a vector of random booleans - let mut booleans = (0..rng.gen_range(1..=3)) + let mut booleans = (0..rng.gen_range(0..=3)) .map(|_| rng.gen_bool(0.5)) .collect::>(); @@ -190,7 +190,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { // An OR for false requires each of its children to be false if *predicate_value { // Create a vector of random booleans - let mut booleans = (0..rng.gen_range(1..=3)) + let mut booleans = (0..rng.gen_range(0..=3)) .map(|_| rng.gen_bool(0.5)) .collect::>(); let len = booleans.len(); @@ -207,7 +207,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { ) } else { Predicate::Or( - (0..rng.gen_range(1..=3)) + (0..rng.gen_range(0..=3)) .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, false)).0) .collect(), ) @@ -245,3 +245,17 @@ impl ArbitraryFrom<(&str, &Value)> for Predicate { ) } } + +impl ArbitraryFrom<(&Table, &Predicate)> for Predicate { + fn arbitrary_from(rng: &mut R, (t, p): &(&Table, &Predicate)) -> Self { + if rng.gen_bool(0.5) { + // produce a true predicate + let p_t = CompoundPredicate::arbitrary_from(rng, &(*t, true)).0; + Predicate::And(vec![p_t, (*p).clone()]) + } else { + // produce a false predicate + let p_f = CompoundPredicate::arbitrary_from(rng, &(*t, false)).0; + Predicate::Or(vec![p_f, (*p).clone()]) + } + } +} diff --git a/simulator/main.rs b/simulator/main.rs index 5491c5780a..a4e99f6ea8 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,24 +1,25 @@ #![allow(clippy::arc_with_non_send_sync, dead_code)] use clap::Parser; use core::panic; -use generation::pick_index; -use generation::plan::{Interaction, InteractionPlan, ResultSet}; -use limbo_core::{Database, LimboError, Result}; -use model::table::Value; +use generation::plan::{InteractionPlan, InteractionPlanState}; +use limbo_core::Database; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use runner::cli::SimulatorCLI; use runner::env::{SimConnection, SimulatorEnv, SimulatorOpts}; +use runner::execution::{execute_plans, Execution, ExecutionHistory, ExecutionResult}; use runner::io::SimulatorIO; +use std::any::Any; use std::backtrace::Backtrace; use std::io::Write; use std::path::Path; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use tempfile::TempDir; mod generation; mod model; mod runner; +mod shrink; fn main() { let _ = env_logger::try_init(); @@ -36,15 +37,30 @@ fn main() { }; let db_path = output_dir.join("simulator.db"); + let doublecheck_db_path = db_path.with_extension("_doublecheck.db"); + let shrunk_db_path = db_path.with_extension("_shrink.db"); + let plan_path = output_dir.join("simulator.plan"); + let shrunk_plan_path = plan_path.with_extension("_shrunk.plan"); + let history_path = output_dir.join("simulator.history"); // Print the seed, the locations of the database and the plan file log::info!("database path: {:?}", db_path); + if cli_opts.doublecheck { + log::info!("doublecheck database path: {:?}", doublecheck_db_path); + } else if cli_opts.shrink { + log::info!("shrunk database path: {:?}", shrunk_db_path); + } log::info!("simulator plan path: {:?}", plan_path); + if cli_opts.shrink { + log::info!("shrunk plan path: {:?}", shrunk_plan_path); + } log::info!("simulator history path: {:?}", history_path); log::info!("seed: {}", seed); + let last_execution = Arc::new(Mutex::new(Execution::new(0, 0, 0))); + std::panic::set_hook(Box::new(move |info| { log::error!("panic occurred"); @@ -61,110 +77,258 @@ fn main() { log::error!("captured backtrace:\n{}", bt); })); - let result = std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path)); + let result = SandboxedResult::from( + std::panic::catch_unwind(|| { + run_simulation( + seed, + &cli_opts, + &db_path, + &plan_path, + last_execution.clone(), + None, + ) + }), + last_execution.clone(), + ); if cli_opts.doublecheck { - // Move the old database and plan file to a new location - let old_db_path = db_path.with_extension("_old.db"); - let old_plan_path = plan_path.with_extension("_old.plan"); - - std::fs::rename(&db_path, &old_db_path).unwrap(); - std::fs::rename(&plan_path, &old_plan_path).unwrap(); - // Run the simulation again - let result2 = - std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path)); + let result2 = SandboxedResult::from( + std::panic::catch_unwind(|| { + run_simulation( + seed, + &cli_opts, + &doublecheck_db_path, + &plan_path, + last_execution.clone(), + None, + ) + }), + last_execution.clone(), + ); match (result, result2) { - (Ok(ExecutionResult { error: None, .. }), Err(_)) => { + (SandboxedResult::Correct, SandboxedResult::Panicked { .. }) => { log::error!("doublecheck failed! first run succeeded, but second run panicked."); } - (Ok(ExecutionResult { error: Some(_), .. }), Err(_)) => { + (SandboxedResult::FoundBug { .. }, SandboxedResult::Panicked { .. }) => { log::error!( - "doublecheck failed! first run failed assertion, but second run panicked." + "doublecheck failed! first run failed an assertion, but second run panicked." ); } - (Err(_), Ok(ExecutionResult { error: None, .. })) => { + (SandboxedResult::Panicked { .. }, SandboxedResult::Correct) => { log::error!("doublecheck failed! first run panicked, but second run succeeded."); } - (Err(_), Ok(ExecutionResult { error: Some(_), .. })) => { + (SandboxedResult::Panicked { .. }, SandboxedResult::FoundBug { .. }) => { log::error!( - "doublecheck failed! first run panicked, but second run failed assertion." + "doublecheck failed! first run panicked, but second run failed an assertion." ); } - ( - Ok(ExecutionResult { error: None, .. }), - Ok(ExecutionResult { error: Some(_), .. }), - ) => { + (SandboxedResult::Correct, SandboxedResult::FoundBug { .. }) => { log::error!( - "doublecheck failed! first run succeeded, but second run failed assertion." + "doublecheck failed! first run succeeded, but second run failed an assertion." ); } - ( - Ok(ExecutionResult { error: Some(_), .. }), - Ok(ExecutionResult { error: None, .. }), - ) => { + (SandboxedResult::FoundBug { .. }, SandboxedResult::Correct) => { log::error!( - "doublecheck failed! first run failed assertion, but second run succeeded." + "doublecheck failed! first run failed an assertion, but second run succeeded." ); } - (Err(_), Err(_)) | (Ok(_), Ok(_)) => { + (SandboxedResult::Correct, SandboxedResult::Correct) + | (SandboxedResult::FoundBug { .. }, SandboxedResult::FoundBug { .. }) + | (SandboxedResult::Panicked { .. }, SandboxedResult::Panicked { .. }) => { // Compare the two database files byte by byte - let old_db = std::fs::read(&old_db_path).unwrap(); - let new_db = std::fs::read(&db_path).unwrap(); - if old_db != new_db { + let db_bytes = std::fs::read(&db_path).unwrap(); + let doublecheck_db_bytes = std::fs::read(&doublecheck_db_path).unwrap(); + if db_bytes != doublecheck_db_bytes { log::error!("doublecheck failed! database files are different."); } else { log::info!("doublecheck succeeded! database files are the same."); } } } + } else { + // No doublecheck, run shrinking if panicking or found a bug. + match &result { + SandboxedResult::Correct => { + log::info!("simulation succeeded"); + } + SandboxedResult::Panicked { + error, + last_execution, + } + | SandboxedResult::FoundBug { + error, + last_execution, + .. + } => { + if let SandboxedResult::FoundBug { history, .. } = &result { + // No panic occurred, so write the history to a file + let f = std::fs::File::create(&history_path).unwrap(); + let mut f = std::io::BufWriter::new(f); + for execution in history.history.iter() { + writeln!( + f, + "{} {} {}", + execution.connection_index, + execution.interaction_index, + execution.secondary_index + ) + .unwrap(); + } + } - // Move the new database and plan file to a new location - let new_db_path = db_path.with_extension("_double.db"); - let new_plan_path = plan_path.with_extension("_double.plan"); - - std::fs::rename(&db_path, &new_db_path).unwrap(); - std::fs::rename(&plan_path, &new_plan_path).unwrap(); - - // Move the old database and plan file back - std::fs::rename(&old_db_path, &db_path).unwrap(); - std::fs::rename(&old_plan_path, &plan_path).unwrap(); - } else if let Ok(result) = result { - // No panic occurred, so write the history to a file - let f = std::fs::File::create(&history_path).unwrap(); - let mut f = std::io::BufWriter::new(f); - for execution in result.history.history.iter() { - writeln!( - f, - "{} {} {}", - execution.connection_index, execution.interaction_index, execution.secondary_index - ) - .unwrap(); - } + log::error!("simulation failed: '{}'", error); + + if cli_opts.shrink { + log::info!("Starting to shrink"); + let shrink = Some(last_execution); + let last_execution = Arc::new(Mutex::new(*last_execution)); + + let shrunk = SandboxedResult::from( + std::panic::catch_unwind(|| { + run_simulation( + seed, + &cli_opts, + &shrunk_db_path, + &shrunk_plan_path, + last_execution.clone(), + shrink, + ) + }), + last_execution, + ); + + match (shrunk, &result) { + ( + SandboxedResult::Panicked { error: e1, .. }, + SandboxedResult::Panicked { error: e2, .. }, + ) + | ( + SandboxedResult::FoundBug { error: e1, .. }, + SandboxedResult::FoundBug { error: e2, .. }, + ) => { + if &e1 != e2 { + log::error!( + "shrinking failed, the error was not properly reproduced" + ); + } else { + log::info!("shrinking succeeded"); + } + } + (_, SandboxedResult::Correct) => { + unreachable!("shrinking should never be called on a correct simulation") + } + _ => { + log::error!("shrinking failed, the error was not properly reproduced"); + } + } - match result.error { - None => { - log::info!("simulation completed successfully"); - } - Some(e) => { - log::error!("simulation failed: {:?}", e); + // Write the shrunk plan to a file + let shrunk_plan = std::fs::read(&shrunk_plan_path).unwrap(); + let mut f = std::fs::File::create(&shrunk_plan_path).unwrap(); + f.write_all(&shrunk_plan).unwrap(); + } } } } // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. println!("database path: {:?}", db_path); + if cli_opts.doublecheck { + println!("doublecheck database path: {:?}", doublecheck_db_path); + } else if cli_opts.shrink { + println!("shrunk database path: {:?}", shrunk_db_path); + } println!("simulator plan path: {:?}", plan_path); + if cli_opts.shrink { + println!("shrunk plan path: {:?}", shrunk_plan_path); + } println!("simulator history path: {:?}", history_path); println!("seed: {}", seed); } +fn move_db_and_plan_files(output_dir: &Path) { + let old_db_path = output_dir.join("simulator.db"); + let old_plan_path = output_dir.join("simulator.plan"); + + let new_db_path = output_dir.join("simulator_double.db"); + let new_plan_path = output_dir.join("simulator_double.plan"); + + std::fs::rename(&old_db_path, &new_db_path).unwrap(); + std::fs::rename(&old_plan_path, &new_plan_path).unwrap(); +} + +fn revert_db_and_plan_files(output_dir: &Path) { + let old_db_path = output_dir.join("simulator.db"); + let old_plan_path = output_dir.join("simulator.plan"); + + let new_db_path = output_dir.join("simulator_double.db"); + let new_plan_path = output_dir.join("simulator_double.plan"); + + std::fs::rename(&new_db_path, &old_db_path).unwrap(); + std::fs::rename(&new_plan_path, &old_plan_path).unwrap(); +} + +enum SandboxedResult { + Panicked { + error: String, + last_execution: Execution, + }, + FoundBug { + error: String, + history: ExecutionHistory, + last_execution: Execution, + }, + Correct, +} + +impl SandboxedResult { + fn from( + result: Result>, + last_execution: Arc>, + ) -> Self { + match result { + Ok(ExecutionResult { error: None, .. }) => SandboxedResult::Correct, + Ok(ExecutionResult { error: Some(e), .. }) => { + let error = format!("{:?}", e); + let last_execution = last_execution.lock().unwrap(); + SandboxedResult::Panicked { + error, + last_execution: *last_execution, + } + } + Err(payload) => { + log::error!("panic occurred"); + let err = if let Some(s) = payload.downcast_ref::<&str>() { + log::error!("{}", s); + s.to_string() + } else if let Some(s) = payload.downcast_ref::() { + log::error!("{}", s); + s.to_string() + } else { + log::error!("unknown panic payload"); + "unknown panic payload".to_string() + }; + + last_execution.clear_poison(); + + SandboxedResult::Panicked { + error: err, + last_execution: *last_execution.lock().unwrap(), + } + } + } + } +} + fn run_simulation( seed: u64, cli_opts: &SimulatorCLI, db_path: &Path, plan_path: &Path, + last_execution: Arc>, + shrink: Option<&Execution>, ) -> ExecutionResult { let mut rng = ChaCha8Rng::seed_from_u64(seed); @@ -231,16 +395,34 @@ fn run_simulation( let mut plans = (1..=env.opts.max_connections) .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &mut env)) .collect::>(); + let mut states = plans + .iter() + .map(|_| InteractionPlanState { + stack: vec![], + interaction_pointer: 0, + secondary_pointer: 0, + }) + .collect::>(); + + let plan = if let Some(failing_execution) = shrink { + // todo: for now, we only use 1 connection, so it's safe to use the first plan. + println!("Interactions Before: {}", plans[0].plan.len()); + let shrunk = plans[0].shrink_interaction_plan(failing_execution); + println!("Interactions After: {}", shrunk.plan.len()); + shrunk + } else { + plans[0].clone() + }; let mut f = std::fs::File::create(plan_path).unwrap(); // todo: create a detailed plan file with all the plans. for now, we only use 1 connection, so it's safe to use the first plan. - f.write_all(plans[0].to_string().as_bytes()).unwrap(); + f.write_all(plan.to_string().as_bytes()).unwrap(); - log::info!("{}", plans[0].stats()); + log::info!("{}", plan.stats()); log::info!("Executing database interaction plan..."); - let result = execute_plans(&mut env, &mut plans); + let result = execute_plans(&mut env, &mut plans, &mut states, last_execution); env.io.print_stats(); @@ -248,187 +430,3 @@ fn run_simulation( result } - -struct Execution { - connection_index: usize, - interaction_index: usize, - secondary_index: usize, -} - -impl Execution { - fn new(connection_index: usize, interaction_index: usize, secondary_index: usize) -> Self { - Self { - connection_index, - interaction_index, - secondary_index, - } - } -} - -struct ExecutionHistory { - history: Vec, -} - -impl ExecutionHistory { - fn new() -> Self { - Self { - history: Vec::new(), - } - } -} - -struct ExecutionResult { - history: ExecutionHistory, - error: Option, -} - -impl ExecutionResult { - fn new(history: ExecutionHistory, error: Option) -> Self { - Self { history, error } - } -} - -fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> ExecutionResult { - let mut history = ExecutionHistory::new(); - let now = std::time::Instant::now(); - // todo: add history here by recording which interaction was executed at which tick - for _tick in 0..env.opts.ticks { - // Pick the connection to interact with - let connection_index = pick_index(env.connections.len(), &mut env.rng); - history.history.push(Execution::new( - connection_index, - plans[connection_index].interaction_pointer, - plans[connection_index].secondary_pointer, - )); - // Execute the interaction for the selected connection - match execute_plan(env, connection_index, plans) { - Ok(_) => {} - Err(err) => { - return ExecutionResult::new(history, Some(err)); - } - } - // Check if the maximum time for the simulation has been reached - if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { - return ExecutionResult::new( - history, - Some(limbo_core::LimboError::InternalError( - "maximum time for simulation reached".into(), - )), - ); - } - } - - ExecutionResult::new(history, None) -} - -fn execute_plan( - env: &mut SimulatorEnv, - connection_index: usize, - plans: &mut [InteractionPlan], -) -> Result<()> { - let connection = &env.connections[connection_index]; - let plan = &mut plans[connection_index]; - - if plan.interaction_pointer >= plan.plan.len() { - return Ok(()); - } - - let interaction = &plan.plan[plan.interaction_pointer].interactions[plan.secondary_pointer]; - - if let SimConnection::Disconnected = connection { - log::info!("connecting {}", connection_index); - env.connections[connection_index] = SimConnection::Connected(env.db.connect()); - } else { - match execute_interaction(env, connection_index, interaction, &mut plan.stack) { - Ok(next_execution) => { - log::debug!("connection {} processed", connection_index); - // Move to the next interaction or property - match next_execution { - ExecutionContinuation::NextInteraction => { - if plan.secondary_pointer + 1 - >= plan.plan[plan.interaction_pointer].interactions.len() - { - // If we have reached the end of the interactions for this property, move to the next property - plan.interaction_pointer += 1; - plan.secondary_pointer = 0; - } else { - // Otherwise, move to the next interaction - plan.secondary_pointer += 1; - } - } - ExecutionContinuation::NextProperty => { - // Skip to the next property - plan.interaction_pointer += 1; - plan.secondary_pointer = 0; - } - } - } - Err(err) => { - log::error!("error {}", err); - return Err(err); - } - } - } - - Ok(()) -} - -/// The next point of control flow after executing an interaction. -/// `execute_interaction` uses this type in conjunction with a result, where -/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case -/// indicates the next step in the plan. -enum ExecutionContinuation { - /// Default continuation, execute the next interaction. - NextInteraction, - /// Typically used in the case of preconditions failures, skip to the next property. - NextProperty, -} - -fn execute_interaction( - env: &mut SimulatorEnv, - connection_index: usize, - interaction: &Interaction, - stack: &mut Vec, -) -> Result { - log::info!("executing: {}", interaction); - match interaction { - generation::plan::Interaction::Query(_) => { - let conn = match &mut env.connections[connection_index] { - SimConnection::Connected(conn) => conn, - SimConnection::Disconnected => unreachable!(), - }; - - log::debug!("{}", interaction); - let results = interaction.execute_query(conn); - log::debug!("{:?}", results); - stack.push(results); - } - generation::plan::Interaction::Assertion(_) => { - interaction.execute_assertion(stack, env)?; - stack.clear(); - } - generation::plan::Interaction::Assumption(_) => { - let assumption_result = interaction.execute_assumption(stack, env); - stack.clear(); - - if assumption_result.is_err() { - log::warn!("assumption failed: {:?}", assumption_result); - return Ok(ExecutionContinuation::NextProperty); - } - } - Interaction::Fault(_) => { - interaction.execute_fault(env, connection_index)?; - } - } - - Ok(ExecutionContinuation::NextInteraction) -} - -fn compare_equal_rows(a: &[Vec], b: &[Vec]) { - assert_eq!(a.len(), b.len(), "lengths are different"); - for (r1, r2) in a.iter().zip(b) { - for (v1, v2) in r1.iter().zip(r2) { - assert_eq!(v1, v2, "values are different"); - } - } -} diff --git a/simulator/model/query.rs b/simulator/model/query.rs index 66297b2adb..40d7b7c89b 100644 --- a/simulator/model/query.rs +++ b/simulator/model/query.rs @@ -53,7 +53,7 @@ impl Display for Predicate { } // This type represents the potential queries on the database. -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) enum Query { Create(Create), Select(Select), @@ -61,6 +61,24 @@ pub(crate) enum Query { Delete(Delete), } +impl Query { + pub(crate) fn dependencies(&self) -> Vec { + match self { + Query::Create(_) => vec![], + Query::Select(Select { table, .. }) + | Query::Insert(Insert { table, .. }) + | Query::Delete(Delete { table, .. }) => vec![table.clone()], + } + } + pub(crate) fn uses(&self) -> Vec { + match self { + Query::Create(Create { table }) => vec![table.name.clone()], + Query::Select(Select { table, .. }) + | Query::Insert(Insert { table, .. }) + | Query::Delete(Delete { table, .. }) => vec![table.clone()], + } + } +} #[derive(Debug, Clone)] pub(crate) struct Create { pub(crate) table: Table, diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 8ad42c8b37..6b56b9b825 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -35,4 +35,10 @@ pub struct SimulatorCLI { default_value_t = 60 * 60 // default to 1 hour )] pub maximum_time: usize, + #[clap( + short = 'm', + long, + help = "minimize(shrink) the failing counterexample" + )] + pub shrink: bool, } diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs new file mode 100644 index 0000000000..064542805a --- /dev/null +++ b/simulator/runner/execution.rs @@ -0,0 +1,202 @@ +use std::sync::{Arc, Mutex}; + +use limbo_core::{LimboError, Result}; + +use crate::generation::{ + self, pick_index, + plan::{Interaction, InteractionPlan, InteractionPlanState, ResultSet}, +}; + +use super::env::{SimConnection, SimulatorEnv}; + +#[derive(Clone, Copy)] +pub(crate) struct Execution { + pub(crate) connection_index: usize, + pub(crate) interaction_index: usize, + pub(crate) secondary_index: usize, +} + +impl Execution { + pub(crate) fn new( + connection_index: usize, + interaction_index: usize, + secondary_index: usize, + ) -> Self { + Self { + connection_index, + interaction_index, + secondary_index, + } + } +} + +pub(crate) struct ExecutionHistory { + pub(crate) history: Vec, +} + +impl ExecutionHistory { + fn new() -> Self { + Self { + history: Vec::new(), + } + } +} + +pub(crate) struct ExecutionResult { + pub(crate) history: ExecutionHistory, + pub(crate) error: Option, +} + +impl ExecutionResult { + fn new(history: ExecutionHistory, error: Option) -> Self { + Self { history, error } + } +} + +pub(crate) fn execute_plans( + env: &mut SimulatorEnv, + plans: &mut [InteractionPlan], + states: &mut [InteractionPlanState], + last_execution: Arc>, +) -> ExecutionResult { + let mut history = ExecutionHistory::new(); + let now = std::time::Instant::now(); + for _tick in 0..env.opts.ticks { + // Pick the connection to interact with + let connection_index = pick_index(env.connections.len(), &mut env.rng); + let state = &mut states[connection_index]; + + history.history.push(Execution::new( + connection_index, + state.interaction_pointer, + state.secondary_pointer, + )); + let mut last_execution = last_execution.lock().unwrap(); + last_execution.connection_index = connection_index; + last_execution.interaction_index = state.interaction_pointer; + last_execution.secondary_index = state.secondary_pointer; + // Execute the interaction for the selected connection + match execute_plan(env, connection_index, plans, states) { + Ok(_) => {} + Err(err) => { + return ExecutionResult::new(history, Some(err)); + } + } + // Check if the maximum time for the simulation has been reached + if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { + return ExecutionResult::new( + history, + Some(limbo_core::LimboError::InternalError( + "maximum time for simulation reached".into(), + )), + ); + } + } + + ExecutionResult::new(history, None) +} + +fn execute_plan( + env: &mut SimulatorEnv, + connection_index: usize, + plans: &mut [InteractionPlan], + states: &mut [InteractionPlanState], +) -> Result<()> { + let connection = &env.connections[connection_index]; + let plan = &mut plans[connection_index]; + let state = &mut states[connection_index]; + + if state.interaction_pointer >= plan.plan.len() { + return Ok(()); + } + + let interaction = &plan.plan[state.interaction_pointer].interactions()[state.secondary_pointer]; + + if let SimConnection::Disconnected = connection { + log::info!("connecting {}", connection_index); + env.connections[connection_index] = SimConnection::Connected(env.db.connect()); + } else { + match execute_interaction(env, connection_index, interaction, &mut state.stack) { + Ok(next_execution) => { + log::debug!("connection {} processed", connection_index); + // Move to the next interaction or property + match next_execution { + ExecutionContinuation::NextInteraction => { + if state.secondary_pointer + 1 + >= plan.plan[state.interaction_pointer].interactions().len() + { + // If we have reached the end of the interactions for this property, move to the next property + state.interaction_pointer += 1; + state.secondary_pointer = 0; + } else { + // Otherwise, move to the next interaction + state.secondary_pointer += 1; + } + } + ExecutionContinuation::NextProperty => { + // Skip to the next property + state.interaction_pointer += 1; + state.secondary_pointer = 0; + } + } + } + Err(err) => { + log::error!("error {}", err); + return Err(err); + } + } + } + + Ok(()) +} + +/// The next point of control flow after executing an interaction. +/// `execute_interaction` uses this type in conjunction with a result, where +/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case +/// indicates the next step in the plan. +enum ExecutionContinuation { + /// Default continuation, execute the next interaction. + NextInteraction, + /// Typically used in the case of preconditions failures, skip to the next property. + NextProperty, +} + +fn execute_interaction( + env: &mut SimulatorEnv, + connection_index: usize, + interaction: &Interaction, + stack: &mut Vec, +) -> Result { + log::info!("executing: {}", interaction); + match interaction { + generation::plan::Interaction::Query(_) => { + let conn = match &mut env.connections[connection_index] { + SimConnection::Connected(conn) => conn, + SimConnection::Disconnected => unreachable!(), + }; + + log::debug!("{}", interaction); + let results = interaction.execute_query(conn); + log::debug!("{:?}", results); + stack.push(results); + } + generation::plan::Interaction::Assertion(_) => { + interaction.execute_assertion(stack, env)?; + stack.clear(); + } + generation::plan::Interaction::Assumption(_) => { + let assumption_result = interaction.execute_assumption(stack, env); + stack.clear(); + + if assumption_result.is_err() { + log::warn!("assumption failed: {:?}", assumption_result); + return Ok(ExecutionContinuation::NextProperty); + } + } + Interaction::Fault(_) => { + interaction.execute_fault(env, connection_index)?; + } + } + + Ok(ExecutionContinuation::NextInteraction) +} diff --git a/simulator/runner/mod.rs b/simulator/runner/mod.rs index 10a777fd96..3f014bef00 100644 --- a/simulator/runner/mod.rs +++ b/simulator/runner/mod.rs @@ -1,5 +1,6 @@ pub mod cli; pub mod env; +pub mod execution; #[allow(dead_code)] pub mod file; pub mod io; diff --git a/simulator/shrink/mod.rs b/simulator/shrink/mod.rs new file mode 100644 index 0000000000..7764a5c307 --- /dev/null +++ b/simulator/shrink/mod.rs @@ -0,0 +1 @@ +pub mod plan; diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs new file mode 100644 index 0000000000..87eb90248b --- /dev/null +++ b/simulator/shrink/plan.rs @@ -0,0 +1,28 @@ +use crate::{generation::plan::InteractionPlan, runner::execution::Execution}; + +impl InteractionPlan { + /// Create a smaller interaction plan by deleting a property + pub(crate) fn shrink_interaction_plan(&self, failing_execution: &Execution) -> InteractionPlan { + let mut plan = self.clone(); + let failing_property = &self.plan[failing_execution.interaction_index]; + let depending_tables = failing_property.dependencies(); + + let before = self.plan.len(); + + // Remove all properties after the failing one + plan.plan.truncate(failing_execution.interaction_index + 1); + // Remove all properties that do not use the failing tables + plan.plan + .retain(|p| p.uses().iter().any(|t| depending_tables.contains(t))); + + let after = plan.plan.len(); + + log::info!( + "Shrinking interaction plan from {} to {} properties", + before, + after + ); + + plan + } +} From 82fcc27a58321c071423ef058d4f245947c19b21 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 02:31:19 +0300 Subject: [PATCH 05/11] this commit fixes query generation; - previous query generation method was faulty, producing wrong assertions - this commit adds a new arbitrary_from implementation for predicates - new implementation takes a table and a row, and produces a predicate that would evaluate to true for the row this commit makes small changes to the main for increasing readability --- simulator/generation/property.rs | 5 +- simulator/generation/query.rs | 164 ++++++++++++++++++++++++++++--- simulator/generation/table.rs | 4 +- simulator/main.rs | 114 +++++++++++---------- simulator/model/query.rs | 10 ++ simulator/runner/cli.rs | 15 +++ simulator/shrink/plan.rs | 4 + 7 files changed, 245 insertions(+), 71 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 39956210be..358b5be17d 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -184,10 +184,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop // Select the row let select_query = Select { table: table.name.clone(), - predicate: Predicate::arbitrary_from( - rng, - &(table, &Predicate::Eq(column.name.clone(), value.clone())), - ), + predicate: Predicate::arbitrary_from(rng, &(table, &row)), }; Property::InsertSelect { diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index c99638f6dc..bc71515c1b 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -3,6 +3,7 @@ use crate::generation::{one_of, Arbitrary, ArbitraryFrom}; use crate::model::query::{Create, Delete, Insert, Predicate, Query, Select}; use crate::model::table::{Table, Value}; +use rand::seq::SliceRandom as _; use rand::Rng; use super::{frequency, pick}; @@ -174,7 +175,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { let len = booleans.len(); // Make sure at least one of them is false - if booleans.iter().all(|b| *b) { + if !booleans.is_empty() && booleans.iter().all(|b| *b) { booleans[rng.gen_range(0..len)] = false; } @@ -195,7 +196,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { .collect::>(); let len = booleans.len(); // Make sure at least one of them is true - if booleans.iter().all(|b| !*b) { + if !booleans.is_empty() && booleans.iter().all(|b| !*b) { booleans[rng.gen_range(0..len)] = true; } @@ -246,16 +247,155 @@ impl ArbitraryFrom<(&str, &Value)> for Predicate { } } -impl ArbitraryFrom<(&Table, &Predicate)> for Predicate { - fn arbitrary_from(rng: &mut R, (t, p): &(&Table, &Predicate)) -> Self { - if rng.gen_bool(0.5) { - // produce a true predicate - let p_t = CompoundPredicate::arbitrary_from(rng, &(*t, true)).0; - Predicate::And(vec![p_t, (*p).clone()]) - } else { - // produce a false predicate - let p_f = CompoundPredicate::arbitrary_from(rng, &(*t, false)).0; - Predicate::Or(vec![p_f, (*p).clone()]) +/// Produces a predicate that is true for the provided row in the given table +fn produce_true_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { + // Pick a column + let column_index = rng.gen_range(0..t.columns.len()); + let column = &t.columns[column_index]; + let value = &row[column_index]; + one_of( + vec![ + Box::new(|_| Predicate::Eq(column.name.clone(), value.clone())), + Box::new(|rng| { + let v = loop { + let v = Value::arbitrary_from(rng, &column.column_type); + if &v != value { + break v; + } + }; + Predicate::Neq(column.name.clone(), v) + }), + Box::new(|rng| { + Predicate::Gt(column.name.clone(), LTValue::arbitrary_from(rng, value).0) + }), + Box::new(|rng| { + Predicate::Lt(column.name.clone(), GTValue::arbitrary_from(rng, value).0) + }), + ], + rng, + ) +} + +/// Produces a predicate that is false for the provided row in the given table +fn produce_false_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { + // Pick a column + let column_index = rng.gen_range(0..t.columns.len()); + let column = &t.columns[column_index]; + let value = &row[column_index]; + one_of( + vec![ + Box::new(|_| Predicate::Neq(column.name.clone(), value.clone())), + Box::new(|rng| { + let v = loop { + let v = Value::arbitrary_from(rng, &column.column_type); + if &v != value { + break v; + } + }; + Predicate::Eq(column.name.clone(), v) + }), + Box::new(|rng| { + Predicate::Gt(column.name.clone(), GTValue::arbitrary_from(rng, value).0) + }), + Box::new(|rng| { + Predicate::Lt(column.name.clone(), LTValue::arbitrary_from(rng, value).0) + }), + ], + rng, + ) +} + +impl ArbitraryFrom<(&Table, &Vec)> for Predicate { + fn arbitrary_from(rng: &mut R, (t, row): &(&Table, &Vec)) -> Self { + // We want to produce a predicate that is true for the row + // We can do this by creating several predicates that + // are true, some that are false, combiend them in ways that correspond to the creation of a true predicate + + // Produce some true and false predicates + let mut true_predicates = (1..=rng.gen_range(1..=4)) + .map(|_| produce_true_predicate(rng, &(*t, row))) + .collect::>(); + + let false_predicates = (0..=rng.gen_range(0..=3)) + .map(|_| produce_false_predicate(rng, &(*t, row))) + .collect::>(); + + // Start building a top level predicate from a true predicate + let mut result = true_predicates.pop().unwrap(); + println!("True predicate: {:?}", result); + + let mut predicates = true_predicates + .iter() + .map(|p| (true, p.clone())) + .chain(false_predicates.iter().map(|p| (false, p.clone()))) + .collect::>(); + + predicates.shuffle(rng); + + while !predicates.is_empty() { + // Create a new predicate from at least 1 and at most 3 predicates + let context = + predicates[0..rng.gen_range(0..=usize::min(3, predicates.len()))].to_vec(); + // Shift `predicates` to remove the predicates in the context + predicates = predicates[context.len()..].to_vec(); + + // `result` is true, so we have the following three options to make a true predicate: + // T or F + // T or T + // T and T + + result = one_of( + vec![ + // T or (X1 or X2 or ... or Xn) + Box::new(|_| { + Predicate::Or(vec![ + result.clone(), + Predicate::Or(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + }), + // T or (T1 and T2 and ... and Tn) + Box::new(|_| { + Predicate::Or(vec![ + result.clone(), + Predicate::And(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + }), + // T and T + Box::new(|_| { + // Check if all the predicates in the context are true + if context.iter().all(|(b, _)| *b) { + // T and (X1 or X2 or ... or Xn) + Predicate::And(vec![ + result.clone(), + Predicate::And(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + } + // Check if there is at least one true predicate + else if context.iter().any(|(b, _)| *b) { + // T and (X1 or X2 or ... or Xn) + Predicate::And(vec![ + result.clone(), + Predicate::Or(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + } else { + // T and (X1 or X2 or ... or Xn or TRUE) + Predicate::And(vec![ + result.clone(), + Predicate::Or( + context + .iter() + .map(|(_, p)| p.clone()) + .chain(std::iter::once(Predicate::true_())) + .collect(), + ), + ]) + } + }), + ], + rng, + ); } + + result } } diff --git a/simulator/generation/table.rs b/simulator/generation/table.rs index 179c53436c..b5b898eebf 100644 --- a/simulator/generation/table.rs +++ b/simulator/generation/table.rs @@ -15,7 +15,7 @@ impl Arbitrary for Name { impl Arbitrary for Table { fn arbitrary(rng: &mut R) -> Self { let name = Name::arbitrary(rng).0; - let columns = (1..=rng.gen_range(1..5)) + let columns = (1..=rng.gen_range(1..10)) .map(|_| Column::arbitrary(rng)) .collect(); Table { @@ -83,7 +83,7 @@ impl ArbitraryFrom for LTValue { fn arbitrary_from(rng: &mut R, value: &Value) -> Self { match value { Value::Integer(i) => Self(Value::Integer(rng.gen_range(i64::MIN..*i - 1))), - Value::Float(f) => Self(Value::Float(rng.gen_range(-1e10..*f - 1.0))), + Value::Float(f) => Self(Value::Float(f - rng.gen_range(0.0..1e10))), Value::Text(t) => { // Either shorten the string, or make at least one character smaller and mutate the rest let mut t = t.clone(); diff --git a/simulator/main.rs b/simulator/main.rs index a4e99f6ea8..1728744af2 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -12,7 +12,7 @@ use runner::io::SimulatorIO; use std::any::Any; use std::backtrace::Backtrace; use std::io::Write; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -21,10 +21,48 @@ mod model; mod runner; mod shrink; -fn main() { +struct Paths { + db: PathBuf, + plan: PathBuf, + shrunk_plan: PathBuf, + history: PathBuf, + doublecheck_db: PathBuf, + shrunk_db: PathBuf, +} + +impl Paths { + fn new(output_dir: &Path, shrink: bool, doublecheck: bool) -> Self { + let paths = Paths { + db: PathBuf::from(output_dir).join("simulator.db"), + plan: PathBuf::from(output_dir).join("simulator.plan"), + shrunk_plan: PathBuf::from(output_dir).join("simulator_shrunk.plan"), + history: PathBuf::from(output_dir).join("simulator.history"), + doublecheck_db: PathBuf::from(output_dir).join("simulator_double.db"), + shrunk_db: PathBuf::from(output_dir).join("simulator_shrunk.db"), + }; + + // Print the seed, the locations of the database and the plan file + log::info!("database path: {:?}", paths.db); + if doublecheck { + log::info!("doublecheck database path: {:?}", paths.doublecheck_db); + } else if shrink { + log::info!("shrunk database path: {:?}", paths.shrunk_db); + } + log::info!("simulator plan path: {:?}", paths.plan); + if shrink { + log::info!("shrunk plan path: {:?}", paths.shrunk_plan); + } + log::info!("simulator history path: {:?}", paths.history); + + paths + } +} + +fn main() -> Result<(), String> { let _ = env_logger::try_init(); let cli_opts = SimulatorCLI::parse(); + cli_opts.validate()?; let seed = match cli_opts.seed { Some(seed) => seed, @@ -33,30 +71,10 @@ fn main() { let output_dir = match &cli_opts.output_dir { Some(dir) => Path::new(dir).to_path_buf(), - None => TempDir::new().unwrap().into_path(), + None => TempDir::new().map_err(|e| format!("{:?}", e))?.into_path(), }; - let db_path = output_dir.join("simulator.db"); - let doublecheck_db_path = db_path.with_extension("_doublecheck.db"); - let shrunk_db_path = db_path.with_extension("_shrink.db"); - - let plan_path = output_dir.join("simulator.plan"); - let shrunk_plan_path = plan_path.with_extension("_shrunk.plan"); - - let history_path = output_dir.join("simulator.history"); - - // Print the seed, the locations of the database and the plan file - log::info!("database path: {:?}", db_path); - if cli_opts.doublecheck { - log::info!("doublecheck database path: {:?}", doublecheck_db_path); - } else if cli_opts.shrink { - log::info!("shrunk database path: {:?}", shrunk_db_path); - } - log::info!("simulator plan path: {:?}", plan_path); - if cli_opts.shrink { - log::info!("shrunk plan path: {:?}", shrunk_plan_path); - } - log::info!("simulator history path: {:?}", history_path); + let paths = Paths::new(&output_dir, cli_opts.shrink, cli_opts.doublecheck); log::info!("seed: {}", seed); let last_execution = Arc::new(Mutex::new(Execution::new(0, 0, 0))); @@ -82,8 +100,8 @@ fn main() { run_simulation( seed, &cli_opts, - &db_path, - &plan_path, + &paths.db, + &paths.plan, last_execution.clone(), None, ) @@ -98,8 +116,8 @@ fn main() { run_simulation( seed, &cli_opts, - &doublecheck_db_path, - &plan_path, + &paths.doublecheck_db, + &paths.plan, last_execution.clone(), None, ) @@ -138,8 +156,8 @@ fn main() { | (SandboxedResult::FoundBug { .. }, SandboxedResult::FoundBug { .. }) | (SandboxedResult::Panicked { .. }, SandboxedResult::Panicked { .. }) => { // Compare the two database files byte by byte - let db_bytes = std::fs::read(&db_path).unwrap(); - let doublecheck_db_bytes = std::fs::read(&doublecheck_db_path).unwrap(); + let db_bytes = std::fs::read(&paths.db).unwrap(); + let doublecheck_db_bytes = std::fs::read(&paths.doublecheck_db).unwrap(); if db_bytes != doublecheck_db_bytes { log::error!("doublecheck failed! database files are different."); } else { @@ -164,7 +182,7 @@ fn main() { } => { if let SandboxedResult::FoundBug { history, .. } = &result { // No panic occurred, so write the history to a file - let f = std::fs::File::create(&history_path).unwrap(); + let f = std::fs::File::create(&paths.history).unwrap(); let mut f = std::io::BufWriter::new(f); for execution in history.history.iter() { writeln!( @@ -190,8 +208,8 @@ fn main() { run_simulation( seed, &cli_opts, - &shrunk_db_path, - &shrunk_plan_path, + &paths.shrunk_db, + &paths.shrunk_plan, last_execution.clone(), shrink, ) @@ -225,8 +243,8 @@ fn main() { } // Write the shrunk plan to a file - let shrunk_plan = std::fs::read(&shrunk_plan_path).unwrap(); - let mut f = std::fs::File::create(&shrunk_plan_path).unwrap(); + let shrunk_plan = std::fs::read(&paths.shrunk_plan).unwrap(); + let mut f = std::fs::File::create(&paths.shrunk_plan).unwrap(); f.write_all(&shrunk_plan).unwrap(); } } @@ -234,18 +252,20 @@ fn main() { } // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. - println!("database path: {:?}", db_path); + println!("database path: {:?}", paths.db); if cli_opts.doublecheck { - println!("doublecheck database path: {:?}", doublecheck_db_path); + println!("doublecheck database path: {:?}", paths.doublecheck_db); } else if cli_opts.shrink { - println!("shrunk database path: {:?}", shrunk_db_path); + println!("shrunk database path: {:?}", paths.shrunk_db); } - println!("simulator plan path: {:?}", plan_path); + println!("simulator plan path: {:?}", paths.plan); if cli_opts.shrink { - println!("shrunk plan path: {:?}", shrunk_plan_path); + println!("shrunk plan path: {:?}", paths.shrunk_plan); } - println!("simulator history path: {:?}", history_path); + println!("simulator history path: {:?}", paths.history); println!("seed: {}", seed); + + Ok(()) } fn move_db_and_plan_files(output_dir: &Path) { @@ -346,18 +366,6 @@ fn run_simulation( (create_percent, read_percent, write_percent, delete_percent) }; - if cli_opts.minimum_size < 1 { - panic!("minimum size must be at least 1"); - } - - if cli_opts.maximum_size < 1 { - panic!("maximum size must be at least 1"); - } - - if cli_opts.maximum_size < cli_opts.minimum_size { - panic!("maximum size must be greater than or equal to minimum size"); - } - let opts = SimulatorOpts { ticks: rng.gen_range(cli_opts.minimum_size..=cli_opts.maximum_size), max_connections: 1, // TODO: for now let's use one connection as we didn't implement diff --git a/simulator/model/query.rs b/simulator/model/query.rs index 40d7b7c89b..a111a508f7 100644 --- a/simulator/model/query.rs +++ b/simulator/model/query.rs @@ -12,6 +12,16 @@ pub(crate) enum Predicate { Lt(String, Value), // column < Value } +impl Predicate { + pub(crate) fn true_() -> Self { + Self::And(vec![]) + } + + pub(crate) fn false_() -> Self { + Self::Or(vec![]) + } +} + impl Display for Predicate { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 6b56b9b825..365ad6a777 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -42,3 +42,18 @@ pub struct SimulatorCLI { )] pub shrink: bool, } + +impl SimulatorCLI { + pub fn validate(&self) -> Result<(), String> { + if self.minimum_size < 1 { + return Err("minimum size must be at least 1".to_string()); + } + if self.maximum_size < 1 { + return Err("maximum size must be at least 1".to_string()); + } + if self.minimum_size > self.maximum_size { + return Err("Minimum size cannot be greater than maximum size".to_string()); + } + Ok(()) + } +} diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 87eb90248b..01fe18f48c 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -3,6 +3,10 @@ use crate::{generation::plan::InteractionPlan, runner::execution::Execution}; impl InteractionPlan { /// Create a smaller interaction plan by deleting a property pub(crate) fn shrink_interaction_plan(&self, failing_execution: &Execution) -> InteractionPlan { + // todo: this is a very naive implementation, next steps are; + // - Shrink to multiple values by removing random interactions + // - Shrink properties by removing their extensions, or shrinking their values + let mut plan = self.clone(); let failing_property = &self.plan[failing_execution.interaction_index]; let depending_tables = failing_property.dependencies(); From 13442808dd0a77504cecf1c514f2f521e062d0b1 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 14:35:42 +0300 Subject: [PATCH 06/11] update properties to add extensional interactions between them --- simulator/generation/property.rs | 128 +++++++++++++++++++++++++------ simulator/generation/query.rs | 28 ++++++- simulator/model/query.rs | 20 +++++ simulator/model/table.rs | 16 ++++ simulator/shrink/plan.rs | 22 +++++- 5 files changed, 189 insertions(+), 25 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 358b5be17d..7ae82de308 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -1,6 +1,6 @@ use crate::{ model::{ - query::{Create, Insert, Predicate, Query, Select}, + query::{Create, Delete, Insert, Predicate, Query, Select}, table::Value, }, runner::env::SimulatorEnv, @@ -35,7 +35,7 @@ pub(crate) enum Property { /// The insert query insert: Insert, /// Additional interactions in the middle of the property - interactions: Vec, + queries: Vec, /// The select query select: Select, }, @@ -55,7 +55,7 @@ pub(crate) enum Property { /// The create query create: Create, /// Additional interactions in the middle of the property - interactions: Vec, + queries: Vec, }, } @@ -70,7 +70,7 @@ impl Property { match self { Property::InsertSelect { insert, - interactions: _, // todo: add extensional interactions + queries, select, } => { // Check that the row is there @@ -106,21 +106,34 @@ impl Property { }), }); - vec![ - assumption, - Interaction::Query(Query::Insert(insert.clone())), - Interaction::Query(Query::Select(select.clone())), - assertion, - ] + let mut interactions = Vec::new(); + interactions.push(assumption); + interactions.push(Interaction::Query(Query::Insert(insert.clone()))); + interactions.extend(queries.clone().into_iter().map(Interaction::Query)); + interactions.push(Interaction::Query(Query::Select(select.clone()))); + interactions.push(assertion); + + interactions } Property::DoubleCreateFailure { create, - interactions: _, // todo: add extensional interactions + queries, } => { let table_name = create.table.name.clone(); + + let assumption = Interaction::Assumption(Assertion { + message: "Double-Create-Failure should not be called on an existing table" + .to_string(), + func: Box::new(move |_: &Vec, env: &SimulatorEnv| { + !env.tables.iter().any(|t| t.name == table_name) + }), + }); + let cq1 = Interaction::Query(Query::Create(create.clone())); let cq2 = Interaction::Query(Query::Create(create.clone())); + let table_name = create.table.name.clone(); + let assertion = Interaction::Assertion(Assertion { message: "creating two tables with the name should result in a failure for the second query" @@ -136,13 +149,26 @@ impl Property { }), }); - vec![cq1, cq2, assertion] + let mut interactions = Vec::new(); + interactions.push(assumption); + interactions.push(cq1); + interactions.extend(queries.clone().into_iter().map(Interaction::Query)); + interactions.push(cq2); + interactions.push(assertion); + + interactions } } } } -fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> (f64, f64, f64) { +pub(crate) struct Remaining { + pub(crate) read: f64, + pub(crate) write: f64, + pub(crate) create: f64, +} + +fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - (stats.read_count as f64)) .max(0.0); @@ -153,10 +179,14 @@ fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> (f64, f64, f64) { - (stats.create_count as f64)) .max(0.0); - (remaining_read, remaining_write, remaining_create) + Remaining { + read: remaining_read, + write: remaining_write, + create: remaining_create, + } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { +fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { // Get a random table let table = pick(&env.tables, rng); // Pick a random column @@ -181,6 +211,36 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop values: vec![row.clone()], }; + // Create random queries respecting the constraints + let mut queries = Vec::new(); + // - [x] There will be no errors in the middle interactions. (this constraint is impossible to check, so this is just best effort) + // - [x] The inserted row will not be deleted. + // - [ ] The inserted row will not be updated. (todo: add this constraint once UPDATE is implemented) + // - [ ] The table `t` will not be renamed, dropped, or altered. (todo: add this constraint once ALTER or DROP is implemented) + for _ in 0..rng.gen_range(0..3) { + let query = Query::arbitrary_from(rng, &(table, remaining)); + match &query { + Query::Delete(Delete { + table: t, + predicate, + }) => { + // The inserted row will not be deleted. + if t == &table.name && predicate.test(&row, &table) { + continue; + } + } + Query::Create(Create { table: t }) => { + // There will be no errors in the middle interactions. + // - Creating the same table is an error + if t.name == table.name { + continue; + } + } + _ => (), + } + queries.push(query); + } + // Select the row let select_query = Select { table: table.name.clone(), @@ -189,12 +249,12 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop Property::InsertSelect { insert: insert_query, - interactions: Vec::new(), + queries, select: select_query, } } -fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) -> Property { +fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { // Get a random table let table = pick(&env.tables, rng); // Create the table @@ -202,27 +262,49 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) table: table.clone(), }; + // Create random queries respecting the constraints + let mut queries = Vec::new(); + // The interactions in the middle has the following constraints; + // - [x] There will be no errors in the middle interactions.(best effort) + // - [ ] Table `t` will not be renamed or dropped.(todo: add this constraint once ALTER or DROP is implemented) + for _ in 0..rng.gen_range(0..3) { + let query = Query::arbitrary_from(rng, &(table, remaining)); + match &query { + Query::Create(Create { table: t }) => { + // There will be no errors in the middle interactions. + // - Creating the same table is an error + if t.name == table.name { + continue; + } + } + _ => (), + } + queries.push(query); + } + Property::DoubleCreateFailure { create: create_query, - interactions: Vec::new(), + queries, } } + + impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { fn arbitrary_from( rng: &mut R, (env, stats): &(&SimulatorEnv, &InteractionStats), ) -> Self { - let (remaining_read, remaining_write, remaining_create) = remaining(env, stats); + let remaining_ = remaining(env, stats); frequency( vec![ ( - f64::min(remaining_read, remaining_write), - Box::new(|rng: &mut R| property_insert_select(rng, env)), + f64::min(remaining_.read, remaining_.write), + Box::new(|rng: &mut R| property_insert_select(rng, env, &remaining_)), ), ( - remaining_create / 2.0, - Box::new(|rng: &mut R| property_double_create_failure(rng, env)), + remaining_.create / 2.0, + Box::new(|rng: &mut R| property_double_create_failure(rng, env, &remaining_)), ), ], rng, diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index bc71515c1b..0229a47500 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -6,6 +6,7 @@ use crate::model::table::{Table, Value}; use rand::seq::SliceRandom as _; use rand::Rng; +use super::property::Remaining; use super::{frequency, pick}; impl Arbitrary for Create { @@ -87,6 +88,32 @@ impl ArbitraryFrom for Query { } } +impl ArbitraryFrom<(&Table, &Remaining)> for Query { + fn arbitrary_from(rng: &mut R, (table, remaining): &(&Table, &Remaining)) -> Self { + frequency( + vec![ + ( + remaining.create, + Box::new(|rng| Self::Create(Create::arbitrary(rng))), + ), + ( + remaining.read, + Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![*table]))), + ), + ( + remaining.write, + Box::new(|rng| Self::Insert(Insert::arbitrary_from(rng, table))), + ), + ( + 0.0, + Box::new(|rng| Self::Delete(Delete::arbitrary_from(rng, table))), + ), + ], + rng, + ) + } +} + struct CompoundPredicate(Predicate); struct SimplePredicate(Predicate); @@ -322,7 +349,6 @@ impl ArbitraryFrom<(&Table, &Vec)> for Predicate { // Start building a top level predicate from a true predicate let mut result = true_predicates.pop().unwrap(); - println!("True predicate: {:?}", result); let mut predicates = true_predicates .iter() diff --git a/simulator/model/query.rs b/simulator/model/query.rs index a111a508f7..9138b1988c 100644 --- a/simulator/model/query.rs +++ b/simulator/model/query.rs @@ -20,6 +20,26 @@ impl Predicate { pub(crate) fn false_() -> Self { Self::Or(vec![]) } + + pub(crate) fn test(&self, row: &[Value], table: &Table) -> bool { + let get_value = |name: &str| { + table + .columns + .iter() + .zip(row.iter()) + .find(|(column, _)| column.name == name) + .map(|(_, value)| value) + }; + + match self { + Predicate::And(vec) => vec.iter().all(|p| p.test(row, table)), + Predicate::Or(vec) => vec.iter().any(|p| p.test(row, table)), + Predicate::Eq(column, value) => get_value(column) == Some(value), + Predicate::Neq(column, value) => get_value(column) != Some(value), + Predicate::Gt(column, value) => get_value(column).map(|v| v > value).unwrap_or(false), + Predicate::Lt(column, value) => get_value(column).map(|v| v < value).unwrap_or(false), + } + } } impl Display for Predicate { diff --git a/simulator/model/table.rs b/simulator/model/table.rs index 841ae0023c..ab3b003afa 100644 --- a/simulator/model/table.rs +++ b/simulator/model/table.rs @@ -53,6 +53,22 @@ pub(crate) enum Value { Blob(Vec), } +impl PartialOrd for Value { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Null, Self::Null) => Some(std::cmp::Ordering::Equal), + (Self::Null, _) => Some(std::cmp::Ordering::Less), + (_, Self::Null) => Some(std::cmp::Ordering::Greater), + (Self::Integer(i1), Self::Integer(i2)) => i1.partial_cmp(i2), + (Self::Float(f1), Self::Float(f2)) => f1.partial_cmp(f2), + (Self::Text(t1), Self::Text(t2)) => t1.partial_cmp(t2), + (Self::Blob(b1), Self::Blob(b2)) => b1.partial_cmp(b2), + // todo: add type coercions here + _ => None, + } + } +} + fn to_sqlite_blob(bytes: &[u8]) -> String { format!( "X'{}'", diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 01fe18f48c..2f89a127f8 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -1,4 +1,7 @@ -use crate::{generation::plan::InteractionPlan, runner::execution::Execution}; +use crate::{ + generation::plan::{Interaction, InteractionPlan, Interactions}, + runner::execution::Execution, +}; impl InteractionPlan { /// Create a smaller interaction plan by deleting a property @@ -19,6 +22,23 @@ impl InteractionPlan { plan.plan .retain(|p| p.uses().iter().any(|t| depending_tables.contains(t))); + // Remove the extensional parts of the properties + for interaction in plan.plan.iter_mut() { + if let Interactions::Property(p) = interaction { + match p { + crate::generation::property::Property::InsertSelect { + queries, + .. + } | + crate::generation::property::Property::DoubleCreateFailure { + queries, + .. + } => { + queries.clear(); + } + } + } + } let after = plan.plan.len(); log::info!( From 43f6c344084bd70d680b0fd6ad534e672d354f21 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 14:43:12 +0300 Subject: [PATCH 07/11] fix arbitrary_from ergonomics by removing the implicit reference in the trait signature --- simulator/generation/mod.rs | 2 +- simulator/generation/plan.rs | 12 ++++---- simulator/generation/property.rs | 27 +++++++++-------- simulator/generation/query.rs | 50 ++++++++++++++++---------------- simulator/generation/table.rs | 12 ++++---- simulator/main.rs | 1 + simulator/shrink/plan.rs | 12 +++----- 7 files changed, 57 insertions(+), 59 deletions(-) diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 6107124f00..73eb77e808 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -13,7 +13,7 @@ pub trait Arbitrary { } pub trait ArbitraryFrom { - fn arbitrary_from(rng: &mut R, t: &T) -> Self; + fn arbitrary_from(rng: &mut R, t: T) -> Self; } pub(crate) fn frequency< diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index e286bb34ab..1deeaef421 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -283,10 +283,8 @@ impl InteractionPlan { } } -impl InteractionPlan { - // todo: This is a hack to get around the fact that `ArbitraryFrom` can't take a mutable - // reference of T, so instead write a bespoke function without using the trait system. - pub(crate) fn arbitrary_from(rng: &mut R, env: &mut SimulatorEnv) -> Self { +impl ArbitraryFrom<&mut SimulatorEnv> for InteractionPlan { + fn arbitrary_from(rng: &mut R, env: &mut SimulatorEnv) -> Self { let mut plan = InteractionPlan::new(); let num_interactions = env.opts.max_interactions; @@ -304,7 +302,7 @@ impl InteractionPlan { plan.plan.len(), num_interactions ); - let interactions = Interactions::arbitrary_from(rng, &(env, plan.stats())); + let interactions = Interactions::arbitrary_from(rng, (env, plan.stats())); interactions.shadow(env); plan.plan.push(interactions); @@ -471,7 +469,7 @@ fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { fn arbitrary_from( rng: &mut R, - (env, stats): &(&SimulatorEnv, InteractionStats), + (env, stats): (&SimulatorEnv, InteractionStats), ) -> Self { let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - (stats.read_count as f64)) @@ -489,7 +487,7 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { ( f64::min(remaining_read, remaining_write) + remaining_create, Box::new(|rng: &mut R| { - Interactions::Property(Property::arbitrary_from(rng, &(env, stats))) + Interactions::Property(Property::arbitrary_from(rng, (env, &stats))) }), ), ( diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 7ae82de308..9ea1462e5b 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -115,10 +115,7 @@ impl Property { interactions } - Property::DoubleCreateFailure { - create, - queries, - } => { + Property::DoubleCreateFailure { create, queries } => { let table_name = create.table.name.clone(); let assumption = Interaction::Assumption(Assertion { @@ -186,7 +183,11 @@ fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { +fn property_insert_select( + rng: &mut R, + env: &SimulatorEnv, + remaining: &Remaining, +) -> Property { // Get a random table let table = pick(&env.tables, rng); // Pick a random column @@ -218,7 +219,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaini // - [ ] The inserted row will not be updated. (todo: add this constraint once UPDATE is implemented) // - [ ] The table `t` will not be renamed, dropped, or altered. (todo: add this constraint once ALTER or DROP is implemented) for _ in 0..rng.gen_range(0..3) { - let query = Query::arbitrary_from(rng, &(table, remaining)); + let query = Query::arbitrary_from(rng, (table, remaining)); match &query { Query::Delete(Delete { table: t, @@ -244,7 +245,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaini // Select the row let select_query = Select { table: table.name.clone(), - predicate: Predicate::arbitrary_from(rng, &(table, &row)), + predicate: Predicate::arbitrary_from(rng, (table, &row)), }; Property::InsertSelect { @@ -254,7 +255,11 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaini } } -fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { +fn property_double_create_failure( + rng: &mut R, + env: &SimulatorEnv, + remaining: &Remaining, +) -> Property { // Get a random table let table = pick(&env.tables, rng); // Create the table @@ -268,7 +273,7 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, // - [x] There will be no errors in the middle interactions.(best effort) // - [ ] Table `t` will not be renamed or dropped.(todo: add this constraint once ALTER or DROP is implemented) for _ in 0..rng.gen_range(0..3) { - let query = Query::arbitrary_from(rng, &(table, remaining)); + let query = Query::arbitrary_from(rng, (table, remaining)); match &query { Query::Create(Create { table: t }) => { // There will be no errors in the middle interactions. @@ -288,12 +293,10 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, } } - - impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { fn arbitrary_from( rng: &mut R, - (env, stats): &(&SimulatorEnv, &InteractionStats), + (env, stats): (&SimulatorEnv, &InteractionStats), ) -> Self { let remaining_ = remaining(env, stats); frequency( diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index 0229a47500..8b93fa993c 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -17,7 +17,7 @@ impl Arbitrary for Create { } } -impl ArbitraryFrom> for Select { +impl ArbitraryFrom<&Vec
> for Select { fn arbitrary_from(rng: &mut R, tables: &Vec
) -> Self { let table = pick(tables, rng); Self { @@ -27,7 +27,7 @@ impl ArbitraryFrom> for Select { } } -impl ArbitraryFrom> for Select { +impl ArbitraryFrom<&Vec<&Table>> for Select { fn arbitrary_from(rng: &mut R, tables: &Vec<&Table>) -> Self { let table = pick(tables, rng); Self { @@ -37,7 +37,7 @@ impl ArbitraryFrom> for Select { } } -impl ArbitraryFrom
for Insert { +impl ArbitraryFrom<&Table> for Insert { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { let num_rows = rng.gen_range(1..10); let values: Vec> = (0..num_rows) @@ -56,7 +56,7 @@ impl ArbitraryFrom
for Insert { } } -impl ArbitraryFrom
for Delete { +impl ArbitraryFrom<&Table> for Delete { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { Self { table: table.name.clone(), @@ -65,7 +65,7 @@ impl ArbitraryFrom
for Delete { } } -impl ArbitraryFrom
for Query { +impl ArbitraryFrom<&Table> for Query { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { frequency( vec![ @@ -89,7 +89,7 @@ impl ArbitraryFrom
for Query { } impl ArbitraryFrom<(&Table, &Remaining)> for Query { - fn arbitrary_from(rng: &mut R, (table, remaining): &(&Table, &Remaining)) -> Self { + fn arbitrary_from(rng: &mut R, (table, remaining): (&Table, &Remaining)) -> Self { frequency( vec![ ( @@ -98,7 +98,7 @@ impl ArbitraryFrom<(&Table, &Remaining)> for Query { ), ( remaining.read, - Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![*table]))), + Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![table]))), ), ( remaining.write, @@ -118,7 +118,7 @@ struct CompoundPredicate(Predicate); struct SimplePredicate(Predicate); impl ArbitraryFrom<(&Table, bool)> for SimplePredicate { - fn arbitrary_from(rng: &mut R, (table, predicate_value): &(&Table, bool)) -> Self { + fn arbitrary_from(rng: &mut R, (table, predicate_value): (&Table, bool)) -> Self { // Pick a random column let column_index = rng.gen_range(0..table.columns.len()); let column = &table.columns[column_index]; @@ -182,15 +182,15 @@ impl ArbitraryFrom<(&Table, bool)> for SimplePredicate { } impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { - fn arbitrary_from(rng: &mut R, (table, predicate_value): &(&Table, bool)) -> Self { + fn arbitrary_from(rng: &mut R, (table, predicate_value): (&Table, bool)) -> Self { // Decide if you want to create an AND or an OR Self(if rng.gen_bool(0.7) { // An AND for true requires each of its children to be true // An AND for false requires at least one of its children to be false - if *predicate_value { + if predicate_value { Predicate::And( (0..rng.gen_range(0..=3)) - .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, true)).0) + .map(|_| SimplePredicate::arbitrary_from(rng, (table, true)).0) .collect(), ) } else { @@ -209,14 +209,14 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { Predicate::And( booleans .iter() - .map(|b| SimplePredicate::arbitrary_from(rng, &(*table, *b)).0) + .map(|b| SimplePredicate::arbitrary_from(rng, (table, *b)).0) .collect(), ) } } else { // An OR for true requires at least one of its children to be true // An OR for false requires each of its children to be false - if *predicate_value { + if predicate_value { // Create a vector of random booleans let mut booleans = (0..rng.gen_range(0..=3)) .map(|_| rng.gen_bool(0.5)) @@ -230,13 +230,13 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { Predicate::Or( booleans .iter() - .map(|b| SimplePredicate::arbitrary_from(rng, &(*table, *b)).0) + .map(|b| SimplePredicate::arbitrary_from(rng, (table, *b)).0) .collect(), ) } else { Predicate::Or( (0..rng.gen_range(0..=3)) - .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, false)).0) + .map(|_| SimplePredicate::arbitrary_from(rng, (table, false)).0) .collect(), ) } @@ -244,28 +244,28 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { } } -impl ArbitraryFrom
for Predicate { +impl ArbitraryFrom<&Table> for Predicate { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { let predicate_value = rng.gen_bool(0.5); - CompoundPredicate::arbitrary_from(rng, &(table, predicate_value)).0 + CompoundPredicate::arbitrary_from(rng, (table, predicate_value)).0 } } impl ArbitraryFrom<(&str, &Value)> for Predicate { - fn arbitrary_from(rng: &mut R, (column_name, value): &(&str, &Value)) -> Self { + fn arbitrary_from(rng: &mut R, (column_name, value): (&str, &Value)) -> Self { one_of( vec![ Box::new(|_| Predicate::Eq(column_name.to_string(), (*value).clone())), Box::new(|rng| { Self::Gt( column_name.to_string(), - GTValue::arbitrary_from(rng, *value).0, + GTValue::arbitrary_from(rng, value).0, ) }), Box::new(|rng| { Self::Lt( column_name.to_string(), - LTValue::arbitrary_from(rng, *value).0, + LTValue::arbitrary_from(rng, value).0, ) }), ], @@ -275,7 +275,7 @@ impl ArbitraryFrom<(&str, &Value)> for Predicate { } /// Produces a predicate that is true for the provided row in the given table -fn produce_true_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { +fn produce_true_predicate(rng: &mut R, (t, row): (&Table, &Vec)) -> Predicate { // Pick a column let column_index = rng.gen_range(0..t.columns.len()); let column = &t.columns[column_index]; @@ -304,7 +304,7 @@ fn produce_true_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) } /// Produces a predicate that is false for the provided row in the given table -fn produce_false_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { +fn produce_false_predicate(rng: &mut R, (t, row): (&Table, &Vec)) -> Predicate { // Pick a column let column_index = rng.gen_range(0..t.columns.len()); let column = &t.columns[column_index]; @@ -333,18 +333,18 @@ fn produce_false_predicate(rng: &mut R, (t, row): &(&Table, &Vec) } impl ArbitraryFrom<(&Table, &Vec)> for Predicate { - fn arbitrary_from(rng: &mut R, (t, row): &(&Table, &Vec)) -> Self { + fn arbitrary_from(rng: &mut R, (t, row): (&Table, &Vec)) -> Self { // We want to produce a predicate that is true for the row // We can do this by creating several predicates that // are true, some that are false, combiend them in ways that correspond to the creation of a true predicate // Produce some true and false predicates let mut true_predicates = (1..=rng.gen_range(1..=4)) - .map(|_| produce_true_predicate(rng, &(*t, row))) + .map(|_| produce_true_predicate(rng, (t, row))) .collect::>(); let false_predicates = (0..=rng.gen_range(0..=3)) - .map(|_| produce_false_predicate(rng, &(*t, row))) + .map(|_| produce_false_predicate(rng, (t, row))) .collect::>(); // Start building a top level predicate from a true predicate diff --git a/simulator/generation/table.rs b/simulator/generation/table.rs index b5b898eebf..af5a018f7f 100644 --- a/simulator/generation/table.rs +++ b/simulator/generation/table.rs @@ -45,7 +45,7 @@ impl Arbitrary for ColumnType { } } -impl ArbitraryFrom> for Value { +impl ArbitraryFrom<&Vec<&Value>> for Value { fn arbitrary_from(rng: &mut R, values: &Vec<&Self>) -> Self { if values.is_empty() { return Self::Null; @@ -55,7 +55,7 @@ impl ArbitraryFrom> for Value { } } -impl ArbitraryFrom for Value { +impl ArbitraryFrom<&ColumnType> for Value { fn arbitrary_from(rng: &mut R, column_type: &ColumnType) -> Self { match column_type { ColumnType::Integer => Self::Integer(rng.gen_range(i64::MIN..i64::MAX)), @@ -68,7 +68,7 @@ impl ArbitraryFrom for Value { pub(crate) struct LTValue(pub(crate) Value); -impl ArbitraryFrom> for LTValue { +impl ArbitraryFrom<&Vec<&Value>> for LTValue { fn arbitrary_from(rng: &mut R, values: &Vec<&Value>) -> Self { if values.is_empty() { return Self(Value::Null); @@ -79,7 +79,7 @@ impl ArbitraryFrom> for LTValue { } } -impl ArbitraryFrom for LTValue { +impl ArbitraryFrom<&Value> for LTValue { fn arbitrary_from(rng: &mut R, value: &Value) -> Self { match value { Value::Integer(i) => Self(Value::Integer(rng.gen_range(i64::MIN..*i - 1))), @@ -128,7 +128,7 @@ impl ArbitraryFrom for LTValue { pub(crate) struct GTValue(pub(crate) Value); -impl ArbitraryFrom> for GTValue { +impl ArbitraryFrom<&Vec<&Value>> for GTValue { fn arbitrary_from(rng: &mut R, values: &Vec<&Value>) -> Self { if values.is_empty() { return Self(Value::Null); @@ -139,7 +139,7 @@ impl ArbitraryFrom> for GTValue { } } -impl ArbitraryFrom for GTValue { +impl ArbitraryFrom<&Value> for GTValue { fn arbitrary_from(rng: &mut R, value: &Value) -> Self { match value { Value::Integer(i) => Self(Value::Integer(rng.gen_range(*i..i64::MAX))), diff --git a/simulator/main.rs b/simulator/main.rs index 1728744af2..db4c7955b5 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -2,6 +2,7 @@ use clap::Parser; use core::panic; use generation::plan::{InteractionPlan, InteractionPlanState}; +use generation::ArbitraryFrom; use limbo_core::Database; use rand::prelude::*; use rand_chacha::ChaCha8Rng; diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 2f89a127f8..c97503f653 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -1,5 +1,5 @@ use crate::{ - generation::plan::{Interaction, InteractionPlan, Interactions}, + generation::plan::{InteractionPlan, Interactions}, runner::execution::Execution, }; @@ -26,13 +26,9 @@ impl InteractionPlan { for interaction in plan.plan.iter_mut() { if let Interactions::Property(p) = interaction { match p { - crate::generation::property::Property::InsertSelect { - queries, - .. - } | - crate::generation::property::Property::DoubleCreateFailure { - queries, - .. + crate::generation::property::Property::InsertSelect { queries, .. } + | crate::generation::property::Property::DoubleCreateFailure { + queries, .. } => { queries.clear(); } From c3ea02783d60018dd1737f40794b41359915ad5d Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 15:56:10 +0300 Subject: [PATCH 08/11] - add doc comments to generation traits and functions - remove pick_index from places where it's possible to use pick instead - allow multiple values to be inserted in the insert-select property --- simulator/generation/mod.rs | 22 +++++++++++++++ simulator/generation/property.rs | 46 +++++++++++++++----------------- simulator/generation/table.rs | 23 +++++++++++----- 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 73eb77e808..23775bf0dc 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -8,14 +8,30 @@ pub mod property; pub mod query; pub mod table; +/// Arbitrary trait for generating random values +/// An implementation of arbitrary is assumed to be a uniform sampling of +/// the possible values of the type, with a bias towards smaller values for +/// practicality. pub trait Arbitrary { fn arbitrary(rng: &mut R) -> Self; } +/// ArbitraryFrom trait for generating random values from a given value +/// ArbitraryFrom allows for constructing relations, where the generated +/// value is dependent on the given value. These relations could be constraints +/// such as generating an integer within an interval, or a value that fits in a table, +/// or a predicate satisfying a given table row. pub trait ArbitraryFrom { fn arbitrary_from(rng: &mut R, t: T) -> Self; } +/// Frequency is a helper function for composing different generators with different frequency +/// of occurences. +/// The type signature for the `N` parameter is a bit complex, but it +/// roughly corresponds to a type that can be summed, compared, subtracted and sampled, which are +/// the operations we require for the implementation. +// todo: switch to a simpler type signature that can accomodate all integer and float types, which +// should be enough for our purposes. pub(crate) fn frequency< 'a, T, @@ -38,6 +54,7 @@ pub(crate) fn frequency< unreachable!() } +/// one_of is a helper function for composing different generators with equal probability of occurence. pub(crate) fn one_of<'a, T, R: rand::Rng>( choices: Vec T + 'a>>, rng: &mut R, @@ -46,15 +63,20 @@ pub(crate) fn one_of<'a, T, R: rand::Rng>( choices[index](rng) } +/// pick is a helper function for uniformly picking a random element from a slice pub(crate) fn pick<'a, T, R: rand::Rng>(choices: &'a [T], rng: &mut R) -> &'a T { let index = rng.gen_range(0..choices.len()); &choices[index] } +/// pick_index is typically used for picking an index from a slice to later refer to the element +/// at that index. pub(crate) fn pick_index(choices: usize, rng: &mut R) -> usize { rng.gen_range(0..choices) } +/// gen_random_text uses `anarchist_readable_name_generator_lib` to generate random +/// readable names for tables, columns, text values etc. fn gen_random_text(rng: &mut T) -> String { let big_text = rng.gen_ratio(1, 1000); if big_text { diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 9ea1462e5b..b00d36c57a 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -7,7 +7,7 @@ use crate::{ }; use super::{ - frequency, pick, pick_index, + frequency, pick, plan::{Assertion, Interaction, InteractionStats, ResultSet}, ArbitraryFrom, }; @@ -66,6 +66,9 @@ impl Property { Property::DoubleCreateFailure { .. } => "Double-Create-Failure".to_string(), } } + /// interactions construct a list of interactions, which is an executable representation of the property. + /// the requirement of property -> vec conversion emerges from the need to serialize the property, + /// and `interaction` cannot be serialized directly. pub(crate) fn interactions(&self) -> Vec { match self { Property::InsertSelect { @@ -73,13 +76,16 @@ impl Property { queries, select, } => { - // Check that the row is there - let row = insert - .values - .first() // `.first` is safe, because we know we are inserting a row in the insert select property - .expect("insert query should have at least 1 value") - .clone(); + // Check that the insert query has at least 1 value + assert!( + !insert.values.is_empty(), + "insert query should have at least 1 value" + ); + // Pick a random row within the insert values + let row = pick(&insert.values, &mut rand::thread_rng()).clone(); + + // Assume that the table exists let assumption = Interaction::Assumption(Assertion { message: format!("table {} exists", insert.table), func: Box::new({ @@ -190,26 +196,18 @@ fn property_insert_select( ) -> Property { // Get a random table let table = pick(&env.tables, rng); - // Pick a random column - let column_index = pick_index(table.columns.len(), rng); - let column = &table.columns[column_index].clone(); - // Generate a random value of the column type - let value = Value::arbitrary_from(rng, &column.column_type); - // Create a whole new row - let mut row = Vec::new(); - for (i, column) in table.columns.iter().enumerate() { - if i == column_index { - row.push(value.clone()); - } else { - let value = Value::arbitrary_from(rng, &column.column_type); - row.push(value); - } - } + // Generate rows to insert + let rows = (0..rng.gen_range(1..=5)) + .map(|_| Vec::::arbitrary_from(rng, table)) + .collect::>(); + + // Pick a random row to select + let row = pick(&rows, rng).clone(); - // Insert the row + // Insert the rows let insert_query = Insert { table: table.name.clone(), - values: vec![row.clone()], + values: rows, }; // Create random queries respecting the constraints diff --git a/simulator/generation/table.rs b/simulator/generation/table.rs index af5a018f7f..8e892e2551 100644 --- a/simulator/generation/table.rs +++ b/simulator/generation/table.rs @@ -1,8 +1,6 @@ use rand::Rng; -use crate::generation::{ - gen_random_text, pick, pick_index, readable_name_custom, Arbitrary, ArbitraryFrom, -}; +use crate::generation::{gen_random_text, pick, readable_name_custom, Arbitrary, ArbitraryFrom}; use crate::model::table::{Column, ColumnType, Name, Table, Value}; impl Arbitrary for Name { @@ -45,6 +43,17 @@ impl Arbitrary for ColumnType { } } +impl ArbitraryFrom<&Table> for Vec { + fn arbitrary_from(rng: &mut R, table: &Table) -> Self { + let mut row = Vec::new(); + for column in table.columns.iter() { + let value = Value::arbitrary_from(rng, &column.column_type); + row.push(value); + } + row + } +} + impl ArbitraryFrom<&Vec<&Value>> for Value { fn arbitrary_from(rng: &mut R, values: &Vec<&Self>) -> Self { if values.is_empty() { @@ -74,8 +83,8 @@ impl ArbitraryFrom<&Vec<&Value>> for LTValue { return Self(Value::Null); } - let index = pick_index(values.len(), rng); - Self::arbitrary_from(rng, values[index]) + let value = pick(values, rng); + Self::arbitrary_from(rng, *value) } } @@ -134,8 +143,8 @@ impl ArbitraryFrom<&Vec<&Value>> for GTValue { return Self(Value::Null); } - let index = pick_index(values.len(), rng); - Self::arbitrary_from(rng, values[index]) + let value = pick(values, rng); + Self::arbitrary_from(rng, *value) } } From fb937eff7bad76030a99a59c6c3d40cdfe0a903c Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 17:26:23 +0300 Subject: [PATCH 09/11] fix non-determinism bug arising from a call to `thread_rng` while picking which row to check existence for in the result of the select query --- simulator/generation/property.rs | 11 ++++++++--- simulator/main.rs | 5 +++-- simulator/runner/execution.rs | 3 ++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index b00d36c57a..cc02e6233d 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -7,7 +7,7 @@ use crate::{ }; use super::{ - frequency, pick, + frequency, pick, pick_index, plan::{Assertion, Interaction, InteractionStats, ResultSet}, ArbitraryFrom, }; @@ -34,6 +34,8 @@ pub(crate) enum Property { InsertSelect { /// The insert query insert: Insert, + /// Selected row index + row_index: usize, /// Additional interactions in the middle of the property queries: Vec, /// The select query @@ -73,6 +75,7 @@ impl Property { match self { Property::InsertSelect { insert, + row_index, queries, select, } => { @@ -83,7 +86,7 @@ impl Property { ); // Pick a random row within the insert values - let row = pick(&insert.values, &mut rand::thread_rng()).clone(); + let row = insert.values[*row_index].clone(); // Assume that the table exists let assumption = Interaction::Assumption(Assertion { @@ -202,7 +205,8 @@ fn property_insert_select( .collect::>(); // Pick a random row to select - let row = pick(&rows, rng).clone(); + let row_index = pick_index(rows.len(), rng).clone(); + let row = rows[row_index].clone(); // Insert the rows let insert_query = Insert { @@ -248,6 +252,7 @@ fn property_insert_select( Property::InsertSelect { insert: insert_query, + row_index, queries, select: select_query, } diff --git a/simulator/main.rs b/simulator/main.rs index db4c7955b5..39a6096c5d 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -218,7 +218,7 @@ fn main() -> Result<(), String> { last_execution, ); - match (shrunk, &result) { + match (&shrunk, &result) { ( SandboxedResult::Panicked { error: e1, .. }, SandboxedResult::Panicked { error: e2, .. }, @@ -227,7 +227,7 @@ fn main() -> Result<(), String> { SandboxedResult::FoundBug { error: e1, .. }, SandboxedResult::FoundBug { error: e2, .. }, ) => { - if &e1 != e2 { + if e1 != e2 { log::error!( "shrinking failed, the error was not properly reproduced" ); @@ -291,6 +291,7 @@ fn revert_db_and_plan_files(output_dir: &Path) { std::fs::rename(&new_plan_path, &old_plan_path).unwrap(); } +#[derive(Debug)] enum SandboxedResult { Panicked { error: String, diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs index 064542805a..3ac44e8949 100644 --- a/simulator/runner/execution.rs +++ b/simulator/runner/execution.rs @@ -9,7 +9,7 @@ use crate::generation::{ use super::env::{SimConnection, SimulatorEnv}; -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] pub(crate) struct Execution { pub(crate) connection_index: usize, pub(crate) interaction_index: usize, @@ -30,6 +30,7 @@ impl Execution { } } +#[derive(Debug)] pub(crate) struct ExecutionHistory { pub(crate) history: Vec, } From c446e29a5087f4efaffa2c9d8d6c16188d3c2e77 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Wed, 15 Jan 2025 11:42:48 +0300 Subject: [PATCH 10/11] add missed updates from the merge --- simulator/generation/plan.rs | 38 ++++++++++++++++++-------------- simulator/generation/property.rs | 2 +- simulator/shrink/plan.rs | 5 +++++ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 96719f0367..01c594648a 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -12,7 +12,10 @@ use crate::{ use crate::generation::{frequency, Arbitrary, ArbitraryFrom}; -use super::{pick, property::Property}; +use super::{ + pick, + property::{remaining, Property}, +}; pub(crate) type ResultSet = Result>>; @@ -470,38 +473,39 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { rng: &mut R, (env, stats): (&SimulatorEnv, InteractionStats), ) -> Self { - let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - - (stats.read_count as f64)) - .max(0.0); - let remaining_write = ((env.opts.max_interactions as f64 * env.opts.write_percent / 100.0) - - (stats.write_count as f64)) - .max(0.0); - let remaining_create = ((env.opts.max_interactions as f64 * env.opts.create_percent - / 100.0) - - (stats.create_count as f64)) - .max(0.0); - + let remaining_ = remaining(env, &stats); + println!( + "remaining: {} {} {}", + remaining_.read, remaining_.write, remaining_.create + ); frequency( vec![ ( - f64::min(remaining_read, remaining_write) + remaining_create, + f64::min(remaining_.read, remaining_.write) + remaining_.create, Box::new(|rng: &mut R| { Interactions::Property(Property::arbitrary_from(rng, (env, &stats))) }), ), ( - remaining_read, + remaining_.read, Box::new(|rng: &mut R| random_read(rng, env)), ), ( - remaining_write, + remaining_.write, Box::new(|rng: &mut R| random_write(rng, env)), ), ( - remaining_create, + remaining_.create, Box::new(|rng: &mut R| create_table(rng, env)), ), - (1.0, Box::new(|rng: &mut R| random_fault(rng, env))), + ( + remaining_ + .read + .min(remaining_.write) + .min(remaining_.create) + .max(1.0), + Box::new(|rng: &mut R| random_fault(rng, env)), + ), ], rng, ) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index cc02e6233d..dd92a7af83 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -174,7 +174,7 @@ pub(crate) struct Remaining { pub(crate) create: f64, } -fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { +pub(crate) fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - (stats.read_count as f64)) .max(0.0); diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index c97503f653..92867d82ec 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -1,5 +1,6 @@ use crate::{ generation::plan::{InteractionPlan, Interactions}, + model::query::Query, runner::execution::Execution, }; @@ -35,6 +36,10 @@ impl InteractionPlan { } } } + + plan.plan + .retain(|p| !matches!(p, Interactions::Query(Query::Select(_)))); + let after = plan.plan.len(); log::info!( From ea6ad8d414a0afe9515de9745c6606856a6e59a6 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Wed, 15 Jan 2025 12:44:43 +0300 Subject: [PATCH 11/11] remove debug print --- simulator/generation/plan.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 01c594648a..028da2a302 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -474,10 +474,6 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { (env, stats): (&SimulatorEnv, InteractionStats), ) -> Self { let remaining_ = remaining(env, &stats); - println!( - "remaining: {} {} {}", - remaining_.read, remaining_.write, remaining_.create - ); frequency( vec![ (