Test runner.

ast/src/parsed/types.rs

@@ -416,6 +416,12 @@ impl<R: Display> From<FunctionType<Expression<R>>> for FunctionType<u64> {
     }
 }
 
+impl From<FunctionType> for Type {
+    fn from(value: FunctionType) -> Self {
+        Type::Function(value)
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize, JsonSchema)]
 pub struct TypeScheme<E = u64> {
     /// Type variables and their trait bounds.

cli/Cargo.toml

@@ -24,6 +24,7 @@ powdr.workspace = true
 
 clap = { version = "^4.3", features = ["derive"] }
 env_logger = "0.10.0"
+itertools = "0.13"
 log = "0.4.17"
 strum = { version = "0.24.1", features = ["derive"] }
 clap-markdown = "0.1.3"

cli/src/main.rs

@@ -12,6 +12,7 @@ use powdr::number::{
     BabyBearField, BigUint, Bn254Field, FieldElement, GoldilocksField, KoalaBearField,
     Mersenne31Field,
 };
+use powdr::pipeline::test_runner;
 use powdr::Pipeline;
 use std::io;
 use std::path::PathBuf;
@@ -363,6 +364,19 @@ enum Commands {
         #[arg(value_parser = clap_enum_variants!(FieldArgument))]
         field: FieldArgument,
     },
+
+    /// Executes all functions starting with `test_` in every module called
+    /// `test` (or sub-module thereof) starting from the given module.
+    Test {
+        /// Input file.
+        file: String,
+
+        /// The field to use
+        #[arg(long)]
+        #[arg(default_value_t = FieldArgument::Gl)]
+        #[arg(value_parser = clap_enum_variants!(FieldArgument))]
+        field: FieldArgument,
+    },
 }
 
 fn split_inputs<T: FieldElement>(inputs: &str) -> Vec<T> {
@@ -469,6 +483,9 @@ fn run_command(command: Commands) {
                 csv_mode
             ))
         }
+        Commands::Test { file, field } => {
+            call_with_field!(run_test::<field>(&file))
+        }
         Commands::Prove {
             file,
             dir,
@@ -688,7 +705,12 @@ fn run<F: FieldElement>(
             .compute_proof()
             .unwrap();
     }
+    Ok(())
+}
 
+fn run_test<T: FieldElement>(file: &str) -> Result<(), Vec<String>> {
+    let include_std_tests = false;
+    test_runner::run_from_file::<T>(file, include_std_tests)?;
     Ok(())
 }
 

number/src/traits.rs

@@ -1,4 +1,9 @@
-use std::{fmt, hash::Hash, ops::*, str::FromStr};
+use std::{
+    fmt::{self, Display},
+    hash::Hash,
+    ops::*,
+    str::FromStr,
+};
 
 use num_traits::{ConstOne, ConstZero, One, Zero};
 use schemars::JsonSchema;
@@ -90,6 +95,18 @@ impl KnownField {
     }
 }
 
+impl Display for KnownField {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            KnownField::BabyBearField => write!(f, "BabyBear"),
+            KnownField::KoalaBearField => write!(f, "KoalaBear"),
+            KnownField::Mersenne31Field => write!(f, "Mersenne31"),
+            KnownField::GoldilocksField => write!(f, "Goldilocks"),
+            KnownField::Bn254Field => write!(f, "Bn254"),
+        }
+    }
+}
+
 /// A field element
 pub trait FieldElement:
     'static

pipeline/src/lib.rs

@@ -3,6 +3,7 @@
 #![deny(clippy::print_stdout)]
 
 pub mod pipeline;
+pub mod test_runner;
 pub mod test_util;
 pub mod util;
 pub mod verify;

pipeline/src/test_runner.rs

@@ -0,0 +1,109 @@
+use std::{collections::BTreeSet, path::PathBuf, str::FromStr};
+
+use itertools::Itertools;
+
+use powdr_ast::{
+    analyzed::{Analyzed, FunctionValueDefinition},
+    parsed::{
+        asm::SymbolPath,
+        types::{FunctionType, Type},
+    },
+};
+use powdr_number::FieldElement;
+use powdr_pil_analyzer::evaluator::{self, SymbolLookup};
+
+use crate::Pipeline;
+
+/// Executes all functions in the given file that start with `test_` and are
+/// inside a module called `test` (or a sub-module thereof).
+///
+/// @param include_std_tests: Whether to run the tests inside the standard library.
+pub fn run_from_file<F: FieldElement>(
+    input: &str,
+    include_std_tests: bool,
+) -> Result<usize, Vec<String>> {
+    let mut pipeline = Pipeline::<F>::default().from_file(PathBuf::from(&input));
+
+    let analyzed = pipeline.compute_analyzed_pil()?;
+    run_tests::<F>(analyzed, include_std_tests)
+}
+
+#[allow(clippy::print_stdout)]
+/// Executes all functions in the given file that start with `test_` and are
+/// inside a module called `test` (or a sub-module thereof).
+///
+/// @param include_std_tests: Whether to run the tests inside the standard library.
+///
+/// Returns the number of tests executed.
+pub fn run_tests<F: FieldElement>(
+    analyzed: &Analyzed<F>,
+    include_std_tests: bool,
+) -> Result<usize, Vec<String>> {
+    let mut symbols = evaluator::Definitions {
+        definitions: &analyzed.definitions,
+        solved_impls: &analyzed.solved_impls,
+    };
+
+    let mut errors = vec![];
+    let tests: BTreeSet<&String> = analyzed
+        .definitions
+        .iter()
+        .filter(|(n, _)| {
+            (n.starts_with("test::") || n.contains("::test::")) && n.contains("::test_")
+        })
+        .filter(|(n, _)| include_std_tests || !n.starts_with("std::"))
+        .filter(|(n, _)| SymbolPath::from_str(n).unwrap().name().starts_with("test_"))
+        .sorted_by_key(|(n, _)| *n)
+        .filter_map(|(n, (_, val))| {
+            let Some(FunctionValueDefinition::Expression(f)) = val else {
+                return None;
+            };
+            // Require a plain `->()` type.
+            (f.type_scheme.as_ref().unwrap().ty
+                == (FunctionType {
+                    params: vec![],
+                    value: Box::new(Type::empty_tuple()),
+                })
+                .into())
+            .then_some(n)
+        })
+        .collect();
+    let field_name = F::known_field().map_or_else(
+        || format!("with modulus {}", F::modulus()),
+        |f| f.to_string(),
+    );
+    println!("Running {} tests using field {field_name}...", tests.len());
+    println!("{}", "-".repeat(85));
+    for name in &tests {
+        let name_len = name.len();
+        let padding = if name_len >= 75 {
+            " ".to_string()
+        } else {
+            " ".repeat(76 - name_len)
+        };
+        print!("{name}...");
+        let function = symbols.lookup(name, &None).unwrap();
+        match evaluator::evaluate_function_call::<F>(function, vec![], &mut symbols) {
+            Err(e) => {
+                let msg = e.to_string();
+                println!("{padding}failed\n  {msg}");
+                errors.push((name, msg));
+            }
+            Ok(_) => println!("{padding}ok"),
+        }
+    }
+
+    println!("{}", "-".repeat(85));
+    if errors.is_empty() {
+        println!("All {} tests passed!", tests.len());
+        Ok(tests.len())
+    } else {
+        println!(
+            "Failed tests: {} / {}\n{}",
+            errors.len(),
+            tests.len(),
+            errors.iter().map(|(n, e)| format!("  {n}: {e}")).join("\n")
+        );
+        Err(vec![format!("{} test(s) failed.", errors.len())])
+    }
+}

pipeline/src/test_util.rs

@@ -21,19 +21,6 @@ pub fn resolve_test_file(file_name: &str) -> PathBuf {
     PathBuf::from(format!("../test_data/{file_name}"))
 }
 
-pub fn execute_test_file(
-    file_name: &str,
-    inputs: Vec<GoldilocksField>,
-    external_witness_values: Vec<(String, Vec<GoldilocksField>)>,
-) -> Result<(), Vec<String>> {
-    Pipeline::default()
-        .from_file(resolve_test_file(file_name))
-        .with_prover_inputs(inputs)
-        .add_external_witness_values(external_witness_values)
-        .compute_witness()
-        .map(|_| ())
-}
-
 /// Makes a new pipeline for the given file. All steps until witness generation are
 /// already computed, so that the test can branch off from there, without having to re-compute
 /// these steps.

pipeline/tests/powdr_std.rs

@@ -4,11 +4,11 @@ use powdr_number::{BabyBearField, BigInt, Bn254Field, GoldilocksField};
 
 use powdr_pil_analyzer::evaluator::Value;
 use powdr_pipeline::{
+    test_runner::run_tests,
     test_util::{
-        evaluate_function, evaluate_integer_function, execute_test_file, gen_estark_proof,
-        gen_halo2_proof, make_simple_prepared_pipeline, regular_test,
-        regular_test_without_small_field, std_analyzed, test_halo2, test_pilcom, test_plonky3,
-        BackendVariant,
+        evaluate_function, evaluate_integer_function, gen_estark_proof, gen_halo2_proof,
+        make_simple_prepared_pipeline, regular_test, regular_test_without_small_field,
+        std_analyzed, test_halo2, test_pilcom, test_plonky3, BackendVariant,
     },
     Pipeline,
 };
@@ -396,58 +396,13 @@ fn ff_inv_big() {
 }
 
 #[test]
-fn fp2() {
-    let analyzed = std_analyzed::<GoldilocksField>();
-    evaluate_function(&analyzed, "std::math::fp2::test::add", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::sub", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::mul", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::square", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::inverse", vec![]);
-
-    let analyzed = std_analyzed::<Bn254Field>();
-    evaluate_function(&analyzed, "std::math::fp2::test::add", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::sub", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::mul", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::square", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::inverse", vec![]);
-
-    let analyzed = std_analyzed::<BabyBearField>();
-    evaluate_function(&analyzed, "std::math::fp2::test::add", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::sub", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::mul", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::square", vec![]);
-    evaluate_function(&analyzed, "std::math::fp2::test::inverse", vec![]);
-}
-
-#[test]
-fn fp4() {
-    let analyzed = std_analyzed::<GoldilocksField>();
-    evaluate_function(&analyzed, "std::math::fp4::test::add", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::sub", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::mul", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::inverse", vec![]);
-
-    let analyzed = std_analyzed::<Bn254Field>();
-    evaluate_function(&analyzed, "std::math::fp4::test::add", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::sub", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::mul", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::inverse", vec![]);
-
-    let analyzed = std_analyzed::<BabyBearField>();
-    evaluate_function(&analyzed, "std::math::fp4::test::add", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::sub", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::mul", vec![]);
-    evaluate_function(&analyzed, "std::math::fp4::test::inverse", vec![]);
-}
-
-#[test]
-fn fingerprint() {
-    let analyzed = std_analyzed::<GoldilocksField>();
-    evaluate_function(
-        &analyzed,
-        "std::protocols::fingerprint::test::test_fingerprint",
-        vec![],
-    );
+fn std_tests() {
+    let count1 = run_tests(&std_analyzed::<GoldilocksField>(), true).unwrap();
+    let count2 = run_tests(&std_analyzed::<Bn254Field>(), true).unwrap();
+    let count3 = run_tests(&std_analyzed::<BabyBearField>(), true).unwrap();
+    assert_eq!(count1, count2);
+    assert_eq!(count2, count3);
+    assert!(count1 >= 9);
 }
 
 #[test]
@@ -494,11 +449,6 @@ fn sort() {
         assert_eq!(input_sorted, result);
     }
 }
-#[test]
-fn btree() {
-    let f = "std/btree_test.asm";
-    execute_test_file(f, Default::default(), vec![]).unwrap();
-}
 
 mod reparse {
 

std/math/fp2.asm

@@ -163,7 +163,7 @@ mod test {
     use std::check::assert;
     use std::array::map;
 
-    let add = || {
+    let test_add = || {
         let test_add = |a, b, c| assert(eq_ext(add_ext(a, b), c), || "Wrong addition result");
 
         // Test adding 0
@@ -176,7 +176,7 @@ mod test {
         test_add(Fp2::Fp2(-1, -1), Fp2::Fp2(3, 4), Fp2::Fp2(2, 3))
     };
 
-    let sub = || {
+    let test_sub = || {
         let test_sub = |a, b, c| assert(eq_ext(sub_ext(a, b), c), || "Wrong subtraction result");
 
         // Test subtracting 0
@@ -188,7 +188,7 @@ mod test {
         test_sub(Fp2::Fp2(-1, -1), Fp2::Fp2(0x78000000, 1), Fp2::Fp2(-0x78000000 - 1, -2))
     };
 
-    let mul = || {
+    let test_mul = || {
         let test_mul = |a, b, c| assert(eq_ext(mul_ext(a, b), c), || "Wrong multiplication result");
 
         // Test multiplication by 1
@@ -207,7 +207,7 @@ mod test {
         test_mul(Fp2::Fp2(-1, -2), Fp2::Fp2(-3, 4), Fp2::Fp2(3 - 11 * 8, 6 - 4))
     };
 
-    let square = || {
+    let test_square = || {
         // Tests consistency with mul_ext
         let test_square = |a| assert(eq_ext(mul_ext(a, a), square_ext(a)), || "Wrong squaring result");
 
@@ -219,7 +219,7 @@ mod test {
         test_square(Fp2::Fp2(-1, -2));
     };
 
-    let inverse = || {
+    let test_inverse = || {
         let test_elements = [
             from_base(1),
             Fp2::Fp2(123, 1234),