[skip ci] yolo

crates/cli/src/commands/patterns_test.rs

@@ -37,21 +37,19 @@ pub async fn get_marzano_pattern_test_results(
     let cwd = std::env::current_dir()?;
     let resolver = GritModuleResolver::new(cwd.to_str().unwrap());
 
-    let test_report: DashMap<String, TestReport> = DashMap::new();
-
     let final_results: DashMap<String, Vec<WrappedResult>> = DashMap::new();
     let unformatted_results: DashMap<PatternLanguage, Vec<WrappedResult>> = DashMap::new();
 
     let runtime = Updater::from_current_bin().await?.get_context()?;
 
-    patterns
+    let test_reports = patterns
         .par_iter()
         .enumerate()
         .map(|(index, pattern)| {
             let lang = PatternLanguage::get_language(&pattern.body);
             let chosen_lang = lang.unwrap_or_default();
             if let PatternLanguage::Universal = chosen_lang {
-                return Ok(());
+                return Ok(None);
             }
             let libs = libs.get_language_directory_or_default(lang)?;
             let rich_pattern = resolver
@@ -101,22 +99,20 @@ pub async fn get_marzano_pattern_test_results(
                         }
                         final_results.insert(pattern_name, results);
                     }
-                    Ok(())
+                    Ok(None)
                 }
                 Err(e) => {
-                    if output == OutputMode::Json {
+                    if output == OutputFormat::Json {
                         let report = TestReport {
-                            success: false,
-                            failure_kind: FailureKind::Compilation,
-                            score: 0,
+                            outcome: TestOutcome::CompilationFailure,
                             message: Some(e.to_string()),
+                            samples: vec![],
                         };
-                        println!("{}", serde_json::to_string(&report)?);
-                        return Ok(());
+                        return Ok(Some(report));
                     }
                     // TODO: this is super hacky, replace with thiserror! codes
                     if e.to_string().contains("No pattern found") {
-                        Ok(())
+                        Ok(None)
                     } else {
                         Err(anyhow!(format!(
                             "Failed to compile pattern {}: {}",
@@ -129,6 +125,9 @@ pub async fn get_marzano_pattern_test_results(
         })
         .collect::<Result<Vec<_>>>()?;
 
+    // Filter out the None values
+    let mut test_report = test_reports.into_iter().flatten().collect::<Vec<_>>();
+
     // Now let's attempt formatting the results that need it
     for (lang, lang_results) in unformatted_results.into_iter() {
         let formatted_expected = format_rich_files(
@@ -213,8 +212,28 @@ pub async fn get_marzano_pattern_test_results(
         }
         OutputFormat::Json => {
             // Collect the test reports
+            let mut sample_results = final_results
+                .values()
+                .map(|r| {
+                    let all_pass = r.iter().all(|r| r.result.is_pass());
+                    TestReport {
+                        outcome: if all_pass {
+                            TestOutcome::Success
+                        } else {
+                            TestOutcome::SampleFailure
+                        },
+                        message: if all_pass {
+                            None
+                        } else {
+                            Some("One or more samples failed".to_string())
+                        },
+                        samples: r.iter().map(|r| r.result.clone()).collect(),
+                    }
+                })
+                .collect::<Vec<_>>();
+            test_report.append(&mut sample_results);
 
-            println!("{}", serde_json::to_string(&report)?);
+            println!("{}", serde_json::to_string(&test_report)?);
         }
         _ => {
             bail!("Output format not supported for this command");
@@ -261,26 +280,20 @@ pub(crate) async fn run_patterns_test(
     get_marzano_pattern_test_results(testable_patterns, &libs, arg, flags.into()).await
 }
 
-#[derive(Debug)]
-enum FailureKind {
-    Compilation,
-    SamplesFailed,
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "snake_case")]
+enum TestOutcome {
+    Success,
+    CompilationFailure,
+    SampleFailure,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
 struct TestReport {
-    /// Did all samples pass?
-    success: bool,
-    /// If not, why did the test fail?
-    failure_kind: FailureKind,
-    /// Explanation of the failure
+    outcome: TestOutcome,
     message: Option<String>,
-    /// What is the overall "score" for the test.
-    /// This is a number between 0 and 100, where 100 is a perfect score.
-    /// If the pattern fails to compile, the score is 0.
-    /// Otherwise each sample is scored and averaged.
-    /// Each sample gets 50 points for matching correctly and 50 points for rewriting correctly.
-    score: usize,
+    /// Sample test details
+    samples: Vec<SampleTestResult>,
 }
 
 #[derive(Debug)]

crates/cli_bin/fixtures/plumbing/test_comp.json

@@ -0,0 +1,7 @@
+[{
+  "body": "`foo`d => `bar`",
+  "config": {
+    "samples": []
+  }
+}
+]

crates/gritmodule/src/testing.rs

@@ -118,7 +118,7 @@ pub enum GritTestResultState {
     FailedPattern,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Clone)]
 pub struct SampleTestResult {
     pub matches: Vec<MatchResult>,
     pub state: GritTestResultState,