diff --git a/ast/src/parsed/types.rs b/ast/src/parsed/types.rs index 3a374d054b..49bed0e11d 100644 --- a/ast/src/parsed/types.rs +++ b/ast/src/parsed/types.rs @@ -416,6 +416,12 @@ impl From>> for FunctionType { } } +impl From for Type { + fn from(value: FunctionType) -> Self { + Type::Function(value) + } +} + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize, JsonSchema)] pub struct TypeScheme { /// Type variables and their trait bounds. diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 04cc991000..097ee5b4d2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -24,6 +24,7 @@ powdr.workspace = true clap = { version = "^4.3", features = ["derive"] } env_logger = "0.10.0" +itertools = "0.13" log = "0.4.17" strum = { version = "0.24.1", features = ["derive"] } clap-markdown = "0.1.3" diff --git a/cli/src/main.rs b/cli/src/main.rs index 6668ecb9c9..b426c173d9 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -12,6 +12,7 @@ use powdr::number::{ BabyBearField, BigUint, Bn254Field, FieldElement, GoldilocksField, KoalaBearField, Mersenne31Field, }; +use powdr::pipeline::test_runner; use powdr::Pipeline; use std::io; use std::path::PathBuf; @@ -363,6 +364,19 @@ enum Commands { #[arg(value_parser = clap_enum_variants!(FieldArgument))] field: FieldArgument, }, + + /// Executes all functions starting with `test_` in every module called + /// `test` (or sub-module thereof) starting from the given module. + Test { + /// Input file. + file: String, + + /// The field to use + #[arg(long)] + #[arg(default_value_t = FieldArgument::Gl)] + #[arg(value_parser = clap_enum_variants!(FieldArgument))] + field: FieldArgument, + }, } fn split_inputs(inputs: &str) -> Vec { @@ -469,6 +483,9 @@ fn run_command(command: Commands) { csv_mode )) } + Commands::Test { file, field } => { + call_with_field!(run_test::(&file)) + } Commands::Prove { file, dir, @@ -688,7 +705,12 @@ fn run( .compute_proof() .unwrap(); } + Ok(()) +} +fn run_test(file: &str) -> Result<(), Vec> { + let include_std_tests = false; + test_runner::run_from_file::(file, include_std_tests)?; Ok(()) } diff --git a/number/src/traits.rs b/number/src/traits.rs index 77192f6896..d15672480b 100644 --- a/number/src/traits.rs +++ b/number/src/traits.rs @@ -1,4 +1,9 @@ -use std::{fmt, hash::Hash, ops::*, str::FromStr}; +use std::{ + fmt::{self, Display}, + hash::Hash, + ops::*, + str::FromStr, +}; use num_traits::{ConstOne, ConstZero, One, Zero}; use schemars::JsonSchema; @@ -90,6 +95,18 @@ impl KnownField { } } +impl Display for KnownField { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + KnownField::BabyBearField => write!(f, "BabyBear"), + KnownField::KoalaBearField => write!(f, "KoalaBear"), + KnownField::Mersenne31Field => write!(f, "Mersenne31"), + KnownField::GoldilocksField => write!(f, "Goldilocks"), + KnownField::Bn254Field => write!(f, "Bn254"), + } + } +} + /// A field element pub trait FieldElement: 'static diff --git a/pipeline/src/lib.rs b/pipeline/src/lib.rs index 2e0dd27707..30ecbcbb84 100644 --- a/pipeline/src/lib.rs +++ b/pipeline/src/lib.rs @@ -3,6 +3,7 @@ #![deny(clippy::print_stdout)] pub mod pipeline; +pub mod test_runner; pub mod test_util; pub mod util; pub mod verify; diff --git a/pipeline/src/test_runner.rs b/pipeline/src/test_runner.rs new file mode 100644 index 0000000000..9ab9ceebe1 --- /dev/null +++ b/pipeline/src/test_runner.rs @@ -0,0 +1,109 @@ +use std::{collections::BTreeSet, path::PathBuf, str::FromStr}; + +use itertools::Itertools; + +use powdr_ast::{ + analyzed::{Analyzed, FunctionValueDefinition}, + parsed::{ + asm::SymbolPath, + types::{FunctionType, Type}, + }, +}; +use powdr_number::FieldElement; +use powdr_pil_analyzer::evaluator::{self, SymbolLookup}; + +use crate::Pipeline; + +/// Executes all functions in the given file that start with `test_` and are +/// inside a module called `test` (or a sub-module thereof). +/// +/// @param include_std_tests: Whether to run the tests inside the standard library. +pub fn run_from_file( + input: &str, + include_std_tests: bool, +) -> Result> { + let mut pipeline = Pipeline::::default().from_file(PathBuf::from(&input)); + + let analyzed = pipeline.compute_analyzed_pil()?; + run_tests::(analyzed, include_std_tests) +} + +#[allow(clippy::print_stdout)] +/// Executes all functions in the given file that start with `test_` and are +/// inside a module called `test` (or a sub-module thereof). +/// +/// @param include_std_tests: Whether to run the tests inside the standard library. +/// +/// Returns the number of tests executed. +pub fn run_tests( + analyzed: &Analyzed, + include_std_tests: bool, +) -> Result> { + let mut symbols = evaluator::Definitions { + definitions: &analyzed.definitions, + solved_impls: &analyzed.solved_impls, + }; + + let mut errors = vec![]; + let tests: BTreeSet<&String> = analyzed + .definitions + .iter() + .filter(|(n, _)| { + (n.starts_with("test::") || n.contains("::test::")) && n.contains("::test_") + }) + .filter(|(n, _)| include_std_tests || !n.starts_with("std::")) + .filter(|(n, _)| SymbolPath::from_str(n).unwrap().name().starts_with("test_")) + .sorted_by_key(|(n, _)| *n) + .filter_map(|(n, (_, val))| { + let Some(FunctionValueDefinition::Expression(f)) = val else { + return None; + }; + // Require a plain `->()` type. + (f.type_scheme.as_ref().unwrap().ty + == (FunctionType { + params: vec![], + value: Box::new(Type::empty_tuple()), + }) + .into()) + .then_some(n) + }) + .collect(); + let field_name = F::known_field().map_or_else( + || format!("with modulus {}", F::modulus()), + |f| f.to_string(), + ); + println!("Running {} tests using field {field_name}...", tests.len()); + println!("{}", "-".repeat(85)); + for name in &tests { + let name_len = name.len(); + let padding = if name_len >= 75 { + " ".to_string() + } else { + " ".repeat(76 - name_len) + }; + print!("{name}..."); + let function = symbols.lookup(name, &None).unwrap(); + match evaluator::evaluate_function_call::(function, vec![], &mut symbols) { + Err(e) => { + let msg = e.to_string(); + println!("{padding}failed\n {msg}"); + errors.push((name, msg)); + } + Ok(_) => println!("{padding}ok"), + } + } + + println!("{}", "-".repeat(85)); + if errors.is_empty() { + println!("All {} tests passed!", tests.len()); + Ok(tests.len()) + } else { + println!( + "Failed tests: {} / {}\n{}", + errors.len(), + tests.len(), + errors.iter().map(|(n, e)| format!(" {n}: {e}")).join("\n") + ); + Err(vec![format!("{} test(s) failed.", errors.len())]) + } +} diff --git a/pipeline/src/test_util.rs b/pipeline/src/test_util.rs index 0236b00dc5..10ef64789b 100644 --- a/pipeline/src/test_util.rs +++ b/pipeline/src/test_util.rs @@ -21,19 +21,6 @@ pub fn resolve_test_file(file_name: &str) -> PathBuf { PathBuf::from(format!("../test_data/{file_name}")) } -pub fn execute_test_file( - file_name: &str, - inputs: Vec, - external_witness_values: Vec<(String, Vec)>, -) -> Result<(), Vec> { - Pipeline::default() - .from_file(resolve_test_file(file_name)) - .with_prover_inputs(inputs) - .add_external_witness_values(external_witness_values) - .compute_witness() - .map(|_| ()) -} - /// Makes a new pipeline for the given file. All steps until witness generation are /// already computed, so that the test can branch off from there, without having to re-compute /// these steps. diff --git a/pipeline/tests/powdr_std.rs b/pipeline/tests/powdr_std.rs index 2b655719fb..fc67dc75d0 100644 --- a/pipeline/tests/powdr_std.rs +++ b/pipeline/tests/powdr_std.rs @@ -4,11 +4,11 @@ use powdr_number::{BabyBearField, BigInt, Bn254Field, GoldilocksField}; use powdr_pil_analyzer::evaluator::Value; use powdr_pipeline::{ + test_runner::run_tests, test_util::{ - evaluate_function, evaluate_integer_function, execute_test_file, gen_estark_proof, - gen_halo2_proof, make_simple_prepared_pipeline, regular_test, - regular_test_without_small_field, std_analyzed, test_halo2, test_pilcom, test_plonky3, - BackendVariant, + evaluate_function, evaluate_integer_function, gen_estark_proof, gen_halo2_proof, + make_simple_prepared_pipeline, regular_test, regular_test_without_small_field, + std_analyzed, test_halo2, test_pilcom, test_plonky3, BackendVariant, }, Pipeline, }; @@ -396,58 +396,13 @@ fn ff_inv_big() { } #[test] -fn fp2() { - let analyzed = std_analyzed::(); - evaluate_function(&analyzed, "std::math::fp2::test::add", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::sub", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::mul", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::square", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::inverse", vec![]); - - let analyzed = std_analyzed::(); - evaluate_function(&analyzed, "std::math::fp2::test::add", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::sub", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::mul", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::square", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::inverse", vec![]); - - let analyzed = std_analyzed::(); - evaluate_function(&analyzed, "std::math::fp2::test::add", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::sub", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::mul", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::square", vec![]); - evaluate_function(&analyzed, "std::math::fp2::test::inverse", vec![]); -} - -#[test] -fn fp4() { - let analyzed = std_analyzed::(); - evaluate_function(&analyzed, "std::math::fp4::test::add", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::sub", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::mul", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::inverse", vec![]); - - let analyzed = std_analyzed::(); - evaluate_function(&analyzed, "std::math::fp4::test::add", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::sub", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::mul", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::inverse", vec![]); - - let analyzed = std_analyzed::(); - evaluate_function(&analyzed, "std::math::fp4::test::add", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::sub", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::mul", vec![]); - evaluate_function(&analyzed, "std::math::fp4::test::inverse", vec![]); -} - -#[test] -fn fingerprint() { - let analyzed = std_analyzed::(); - evaluate_function( - &analyzed, - "std::protocols::fingerprint::test::test_fingerprint", - vec![], - ); +fn std_tests() { + let count1 = run_tests(&std_analyzed::(), true).unwrap(); + let count2 = run_tests(&std_analyzed::(), true).unwrap(); + let count3 = run_tests(&std_analyzed::(), true).unwrap(); + assert_eq!(count1, count2); + assert_eq!(count2, count3); + assert!(count1 >= 9); } #[test] @@ -494,11 +449,6 @@ fn sort() { assert_eq!(input_sorted, result); } } -#[test] -fn btree() { - let f = "std/btree_test.asm"; - execute_test_file(f, Default::default(), vec![]).unwrap(); -} mod reparse { diff --git a/std/math/fp2.asm b/std/math/fp2.asm index 1ad18202dc..1d49bfd903 100644 --- a/std/math/fp2.asm +++ b/std/math/fp2.asm @@ -163,7 +163,7 @@ mod test { use std::check::assert; use std::array::map; - let add = || { + let test_add = || { let test_add = |a, b, c| assert(eq_ext(add_ext(a, b), c), || "Wrong addition result"); // Test adding 0 @@ -176,7 +176,7 @@ mod test { test_add(Fp2::Fp2(-1, -1), Fp2::Fp2(3, 4), Fp2::Fp2(2, 3)) }; - let sub = || { + let test_sub = || { let test_sub = |a, b, c| assert(eq_ext(sub_ext(a, b), c), || "Wrong subtraction result"); // Test subtracting 0 @@ -188,7 +188,7 @@ mod test { test_sub(Fp2::Fp2(-1, -1), Fp2::Fp2(0x78000000, 1), Fp2::Fp2(-0x78000000 - 1, -2)) }; - let mul = || { + let test_mul = || { let test_mul = |a, b, c| assert(eq_ext(mul_ext(a, b), c), || "Wrong multiplication result"); // Test multiplication by 1 @@ -207,7 +207,7 @@ mod test { test_mul(Fp2::Fp2(-1, -2), Fp2::Fp2(-3, 4), Fp2::Fp2(3 - 11 * 8, 6 - 4)) }; - let square = || { + let test_square = || { // Tests consistency with mul_ext let test_square = |a| assert(eq_ext(mul_ext(a, a), square_ext(a)), || "Wrong squaring result"); @@ -219,7 +219,7 @@ mod test { test_square(Fp2::Fp2(-1, -2)); }; - let inverse = || { + let test_inverse = || { let test_elements = [ from_base(1), Fp2::Fp2(123, 1234), diff --git a/std/math/fp4.asm b/std/math/fp4.asm index 79a250f256..b23b73479c 100644 --- a/std/math/fp4.asm +++ b/std/math/fp4.asm @@ -146,7 +146,7 @@ mod test { use std::check::assert; use std::array::map; - let add = || { + let test_add = || { let test_add = |a, b, c| assert(eq_ext(add_ext(a, b), c), || "Wrong addition result"); // Test adding 0 @@ -165,7 +165,7 @@ mod test { test_add(Fp4::Fp4(-1, 0, 0, 0), Fp4::Fp4(1, 0, 0, 0), from_base(0)); }; - let sub = || { + let test_sub = || { let test_sub = |a, b, c| assert(eq_ext(sub_ext(a, b), c), || "Wrong subtraction result"); // Test subtracting 0 @@ -177,7 +177,7 @@ mod test { test_sub(Fp4::Fp4(-1, -1, 0, 0), Fp4::Fp4(0x78000000, 1, 0, 0), Fp4::Fp4(-0x78000000 - 1, -2, 0, 0)) }; - let mul = || { + let test_mul = || { let test_mul = |a, b, c| assert(eq_ext(mul_ext(a, b), c), || "Wrong multiplication result"); // Test multiplication by 1 @@ -197,7 +197,7 @@ mod test { test_mul(Fp4::Fp4(-1, -2, -3, -4), Fp4::Fp4(-3, 4, 4, 5), Fp4::Fp4(-415, -339, -223, -13)); }; - let inverse = || { + let test_inverse = || { let test_elements = [ from_base(1), Fp4::Fp4(123, 1234, 1, 2), diff --git a/std/mod.asm b/std/mod.asm index cb3e670db7..9c9f1b35e6 100644 --- a/std/mod.asm +++ b/std/mod.asm @@ -10,5 +10,6 @@ mod math; mod prelude; mod protocols; mod prover; +mod test; mod utils; mod well_known; diff --git a/test_data/std/btree_test.asm b/std/test/btree.asm similarity index 74% rename from test_data/std/btree_test.asm rename to std/test/btree.asm index 3de0ee6441..552b218c76 100644 --- a/test_data/std/btree_test.asm +++ b/std/test/btree.asm @@ -14,21 +14,6 @@ let cmp: int, int -> CmpResult = |a, b| CmpResult::Greater } }; -let b1 = new::(); -let b2 = insert(b1, (1, "a"), cmp); -let b3 = insert(b2, (8, "b"), cmp); -let b4 = insert(b3, (4, "c"), cmp); -let b5 = insert(b4, (2, "d"), cmp); -let b6 = insert(b5, (9, "e"), cmp); -let b7 = insert(b6, (3, "f"), cmp); -let b8 = insert(b7, (7, "g"), cmp); -let b9 = insert(b8, (5, "h"), cmp); -let b10 = insert(b9, (6, "i"), cmp); -let b11 = insert(b10, (0, "j"), cmp); -let b12 = insert(b11, (10, "k"), cmp); -// this one replaces -let b13 = insert(b12, (4, "l"), cmp); -let b14 = insert(b13, (6, "m"), cmp); let one: int = 1; let false: bool = one == 0; @@ -82,7 +67,23 @@ let do_print: BTree -> () = |b_tree| { () }; -machine Main { +let test_btree = || { + let b1 = new::(); + let b2 = insert(b1, (1, "a"), cmp); + let b3 = insert(b2, (8, "b"), cmp); + let b4 = insert(b3, (4, "c"), cmp); + let b5 = insert(b4, (2, "d"), cmp); + let b6 = insert(b5, (9, "e"), cmp); + let b7 = insert(b6, (3, "f"), cmp); + let b8 = insert(b7, (7, "g"), cmp); + let b9 = insert(b8, (5, "h"), cmp); + let b10 = insert(b9, (6, "i"), cmp); + let b11 = insert(b10, (0, "j"), cmp); + let b12 = insert(b11, (10, "k"), cmp); + // this one replaces + let b13 = insert(b12, (4, "l"), cmp); + let b14 = insert(b13, (6, "m"), cmp); + expect(b14, 6, "m"); expect(b13, 6, "i"); expect(b14, 4, "l"); @@ -91,8 +92,8 @@ machine Main { expect(b14, 7, "g"); expect(b14, 0, "j"); expect(b14, 9, "e"); - do_print(b14); + //do_print(b14); let skewed = std::utils::fold(200, |i| i, new::(), |tree, i| insert(tree, (i, ""), cmp)); - do_print(skewed); -} + //do_print(skewed); +}; diff --git a/std/test/mod.asm b/std/test/mod.asm new file mode 100644 index 0000000000..c9ca01e0b8 --- /dev/null +++ b/std/test/mod.asm @@ -0,0 +1 @@ +mod btree;