From c613c729a1120d205faa0d323c2d14299c32c967 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Wed, 4 Oct 2023 23:28:49 +0200 Subject: [PATCH 01/15] Add json checker --- Cargo.toml | 15 ++++++------ config_scheme.json | 28 ++++++++++++++++++++++ src/json.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 9 +++++++ src/report/mod.rs | 2 +- 5 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 src/json.rs diff --git a/Cargo.toml b/Cargo.toml index f637308..74e89d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "havocompare" -description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, text files, pdf-texts and images." +description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, JSON, text files, pdf-texts and images." repository = "https://github.com/VolumeGraphics/havocompare" homepage = "https://github.com/VolumeGraphics/havocompare" documentation = "https://docs.rs/havocompare" -version = "0.4.0" +version = "0.5.0" edition = "2021" license = "MIT" authors = ["Volume Graphics GmbH"] @@ -18,7 +18,7 @@ path = "src/print_args.rs" [dependencies] -clap = {version= "4.3", features=["derive"]} +clap = {version= "4.4", features=["derive"]} chrono = "0.4" serde = "1.0" serde_yaml = "0.9" @@ -27,7 +27,7 @@ schemars_derive = "0.8" thiserror = "1.0" regex = "1.8" image = "0.24" -image-compare = "0.3.0" +image-compare = "0.3.1" tracing = "0.1" tracing-subscriber = "0.3" serde_json = "1.0" @@ -39,14 +39,15 @@ tera = "1.19" sha2 = "0.10" data-encoding = "2.4" permutation = "0.4" -pdf-extract = "0.6" +pdf-extract = "0.7" vg_errortools = "0.1" -rayon = "1.7.0" +rayon = "1.8.0" enable-ansi-support = "0.2" -tempfile = "3.6" +tempfile = "3.8" fs_extra = "1.3" opener = "0.6" anyhow = "1.0" +json_diff = {git = "https://github.com/ChrisRega/json-diff"} [dev-dependencies] env_logger = "0.10" diff --git a/config_scheme.json b/config_scheme.json index 9ad2606..b4c65e1 100644 --- a/config_scheme.json +++ b/config_scheme.json @@ -148,6 +148,21 @@ } } }, + "JsonConfig": { + "description": "configuration for the json compare module", + "type": "object", + "required": [ + "ignore_keys" + ], + "properties": { + "ignore_keys": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "Mode": { "description": "comparison mode for csv cells", "oneOf": [ @@ -457,6 +472,19 @@ }, "additionalProperties": false }, + { + "description": "Compare JSON files", + "type": "object", + "required": [ + "Json" + ], + "properties": { + "Json": { + "$ref": "#/definitions/JsonConfig" + } + }, + "additionalProperties": false + }, { "description": "Run external comparison executable", "type": "object", diff --git a/src/json.rs b/src/json.rs new file mode 100644 index 0000000..a095e07 --- /dev/null +++ b/src/json.rs @@ -0,0 +1,59 @@ +use crate::report::{DiffDetail, Difference}; +use crate::Error; +use itertools::Itertools; +use schemars_derive::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::path::Path; +use tracing::error; + +#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] +/// configuration for the json compare module +pub struct JsonConfig { + ignore_keys: Vec, +} + +pub(crate) fn compare_files>( + nominal: P, + actual: P, + config: &JsonConfig, +) -> Result { + let mut diff = Difference::new_for_file(&nominal, &actual); + let compared_file_name = nominal.as_ref().to_string_lossy().into_owned(); + + let nominal = vg_errortools::fat_io_wrap_std(&nominal, &std::fs::read_to_string)?; + let actual = vg_errortools::fat_io_wrap_std(&actual, &std::fs::read_to_string)?; + + let json_diff = json_diff::process::compare_jsons(&nominal, &actual); + let json_diff = match json_diff { + Ok(diff) => diff, + Err(e) => { + let error_message = + format!("JSON deserialization failed for {compared_file_name} (error: {e})"); + error!("{}", error_message); + diff.push_detail(DiffDetail::Error(error_message)); + diff.error(); + return Ok(diff); + } + }; + let filtered_diff: Vec<_> = json_diff + .all_diffs() + .into_iter() + .filter(|(_d, v)| !config.ignore_keys.contains(v)) + .collect(); + + if !filtered_diff.is_empty() { + let all_diffs_log = filtered_diff.iter().map(|(_, v)| v).join("\n"); + diff.push_detail(DiffDetail::External { + stdout: all_diffs_log, + stderr: String::new(), + }); + + diff.error(); + } + + for (d_type, key) in filtered_diff { + error!("{d_type}: {key}"); + } + + Ok(diff) +} diff --git a/src/lib.rs b/src/lib.rs index da5f668..4a317d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,9 @@ mod pdf; mod properties; mod report; +mod json; +pub use crate::json::JsonConfig; + use crate::external::ExternalConfig; pub use crate::html::HTMLCompareConfig; use crate::properties::PropertiesConfig; @@ -93,6 +96,9 @@ pub enum ComparisonMode { /// Compare file-properties FileProperties(PropertiesConfig), + /// Compare JSON files + Json(JsonConfig), + /// Run external comparison executable External(ExternalConfig), } @@ -194,6 +200,9 @@ fn process_file(nominal: impl AsRef, actual: impl AsRef, rule: &Rule external::compare_files(nominal.as_ref(), actual.as_ref(), conf) .map_err(|e| e.into()) } + ComparisonMode::Json(conf) => { + json::compare_files(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into()) + } } }; let compare_result = match compare_result { diff --git a/src/report/mod.rs b/src/report/mod.rs index 5a2af8a..5c9d9c5 100644 --- a/src/report/mod.rs +++ b/src/report/mod.rs @@ -703,7 +703,7 @@ pub(crate) fn create_html( ) .unwrap_or_else(|e| log_detail_html_creation_error(&e)) } - ComparisonMode::External(_) => { + ComparisonMode::External(_) | ComparisonMode::Json(_) => { if let Some((stdout, stderr)) = file .detail .iter() From fc4115b7d7257ca019d0a1d9f15802350d90ab28 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Wed, 4 Oct 2023 23:50:59 +0200 Subject: [PATCH 02/15] Add json checker --- Cargo.toml | 3 + src/json.rs | 10 ++-- src/report/mod.rs | 60 +++++++++++++++++-- src/report/template.rs | 78 +++++++++++++++++++++++++ tests/integ.rs | 14 +++++ tests/integ/data/json/actual/guy.json | 1 + tests/integ/data/json/expected/guy.json | 1 + tests/integ/json.yml | 7 +++ 8 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 tests/integ/data/json/actual/guy.json create mode 100644 tests/integ/data/json/expected/guy.json create mode 100644 tests/integ/json.yml diff --git a/Cargo.toml b/Cargo.toml index 74e89d1..bf8b82b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,9 @@ opener = "0.6" anyhow = "1.0" json_diff = {git = "https://github.com/ChrisRega/json-diff"} + + + [dev-dependencies] env_logger = "0.10" tracing = {version = "0.1", default-features = false} diff --git a/src/json.rs b/src/json.rs index a095e07..713b470 100644 --- a/src/json.rs +++ b/src/json.rs @@ -42,10 +42,12 @@ pub(crate) fn compare_files>( .collect(); if !filtered_diff.is_empty() { - let all_diffs_log = filtered_diff.iter().map(|(_, v)| v).join("\n"); - diff.push_detail(DiffDetail::External { - stdout: all_diffs_log, - stderr: String::new(), + let all_diffs_log = filtered_diff + .iter() + .map(|(d, v)| format!("{d}: {v}")) + .join("\n"); + diff.push_detail(DiffDetail::Json { + differences: all_diffs_log, }); diff.error(); diff --git a/src/report/mod.rs b/src/report/mod.rs index 5c9d9c5..dbcda29 100644 --- a/src/report/mod.rs +++ b/src/report/mod.rs @@ -119,7 +119,7 @@ impl Difference { return false; } self.is_error |= other.is_error; - self.detail.extend(other.detail.into_iter()); + self.detail.extend(other.detail); true } } @@ -146,6 +146,9 @@ pub enum DiffDetail { stdout: String, stderr: String, }, + Json { + differences: String, + }, Properties(MetaDataPropertyDiff), Error(String), } @@ -265,7 +268,7 @@ pub(crate) fn write_csv_detail( let columns: Vec = n .into_iter() - .zip(a.into_iter()) + .zip(a) .enumerate() .map(|(col, (n, a))| { let current_pos = Position { col, row }; @@ -500,6 +503,34 @@ pub fn write_external_detail( Ok(Some(detail_path)) } +pub fn write_json_detail( + nominal: impl AsRef, + actual: impl AsRef, + differences: &str, + report_dir: impl AsRef, +) -> Result, Error> { + let detail_path = create_detail_folder(report_dir.as_ref())?; + let detail_file = detail_path.path.join(template::DETAIL_FILENAME); + + let mut tera = Tera::default(); + tera.add_raw_template( + &detail_file.to_string_lossy(), + template::PLAIN_JSON_DETAIL_TEMPLATE, + )?; + + let mut ctx = Context::new(); + ctx.insert("actual", &actual.as_ref().to_string_lossy()); + ctx.insert("nominal", &nominal.as_ref().to_string_lossy()); + ctx.insert("differences", differences); + + let file = fat_io_wrap_std(&detail_file, &File::create)?; + debug!("detail html {:?} created", &detail_file); + + tera.render_to(&detail_file.to_string_lossy(), &ctx, file)?; + + Ok(Some(detail_path)) +} + fn create_error_detail( nominal: impl AsRef, actual: impl AsRef, @@ -628,7 +659,7 @@ pub(crate) fn create_html( &file.nominal_file, &file.actual_file, &diffs, - &config, + config, &sub_folder, ) .unwrap_or_else(|e| log_detail_html_creation_error(&e)) @@ -703,7 +734,7 @@ pub(crate) fn create_html( ) .unwrap_or_else(|e| log_detail_html_creation_error(&e)) } - ComparisonMode::External(_) | ComparisonMode::Json(_) => { + ComparisonMode::External(_) => { if let Some((stdout, stderr)) = file .detail .iter() @@ -725,6 +756,27 @@ pub(crate) fn create_html( None } } + ComparisonMode::Json(_) => { + if let Some(differences) = file + .detail + .iter() + .filter_map(|r| match r { + DiffDetail::Json { differences } => Some(differences), + _ => None, + }) + .next() + { + write_json_detail( + &file.nominal_file, + &file.actual_file, + differences, + &sub_folder, + ) + .unwrap_or_else(|e| log_detail_html_creation_error(&e)) + } else { + None + } + } ComparisonMode::FileProperties(_) => None, //we need only additional columns in the index.html ComparisonMode::Hash(_) => { let diffs: Vec = file diff --git a/src/report/template.rs b/src/report/template.rs index 4ea1c98..cafe67d 100644 --- a/src/report/template.rs +++ b/src/report/template.rs @@ -680,6 +680,84 @@ pub const PLAIN_EXTERNAL_DETAIL_TEMPLATE: &str = r###" + + +"###; + +pub const PLAIN_JSON_DETAIL_TEMPLATE: &str = r###" + + + + + Results + + + + + + +

Compare Result of {{ actual }} and {{ nominal }}

+ + + + + + + + + + + + +
Differences
+{{ differences }} +
+ + "###; diff --git a/tests/integ.rs b/tests/integ.rs index 6e4f84c..0c95158 100644 --- a/tests/integ.rs +++ b/tests/integ.rs @@ -36,3 +36,17 @@ fn images_test() { ) .unwrap()); } + +#[test] +fn json_test() { + let report_dir = + tempfile::tempdir().expect("Could not generate temporary directory for report"); + + assert!(!compare_folders( + "tests/integ/data/json/expected/", + "tests/integ/data/json/actual/", + "tests/integ/json.yml", + report_dir + ) + .unwrap()); +} diff --git a/tests/integ/data/json/actual/guy.json b/tests/integ/data/json/actual/guy.json new file mode 100644 index 0000000..c3fb985 --- /dev/null +++ b/tests/integ/data/json/actual/guy.json @@ -0,0 +1 @@ +{"name":"Takumi", "age":18, "car": "Panda Trueno"} \ No newline at end of file diff --git a/tests/integ/data/json/expected/guy.json b/tests/integ/data/json/expected/guy.json new file mode 100644 index 0000000..74ac4c1 --- /dev/null +++ b/tests/integ/data/json/expected/guy.json @@ -0,0 +1 @@ +{"name":"Keisuke", "age":21, "car": "RX7", "brothers": ["Ryosuke"]} \ No newline at end of file diff --git a/tests/integ/json.yml b/tests/integ/json.yml new file mode 100644 index 0000000..b771bfb --- /dev/null +++ b/tests/integ/json.yml @@ -0,0 +1,7 @@ +rules: + - name: "Compare JSON files" + pattern_include: + - "**/*.json" + Json: + ignore_keys: + - "" \ No newline at end of file From 3c7507f7ea57f9fb13b8d81a97aead056282a338 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Wed, 4 Oct 2023 23:59:25 +0200 Subject: [PATCH 03/15] Add contributors to README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index a885702..87ab80e 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,11 @@ [![Coverage Status](https://coveralls.io/repos/github/VolumeGraphics/havocompare/badge.svg?branch=main)](https://coveralls.io/github/VolumeGraphics/havocompare?branch=main) [![License](https://img.shields.io/badge/license-MIT-blue?style=flat)](LICENSE) +## Contributors: + + Contributors + + ## Quickstart ### 0. Install havocompare From 1fc2c772d8289aa91b6f857046c25491659ce5fd Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 5 Oct 2023 08:23:44 +0200 Subject: [PATCH 04/15] Fix README.md with example --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 87ab80e..7f236a6 100644 --- a/README.md +++ b/README.md @@ -226,8 +226,28 @@ rules: - "--only-images" ``` + +#### JSON comparison +Compares JSON files for different keys in both files and mismatches in values. +ignore_keys elements will be ignored, no substring or regex is supported here. +If you need this, please file a bug. + +```yaml +rules: +- name: "Compare JSON files" + pattern_include: + - "**/*.json" + Json: + ignore_keys: + - "" +``` + + ## Changelog +### 0.5.0 +- Add basic JSON checking + ### 0.4.0 - Separate reporting logic from comparison logic - Implement a machine-readable JSON reporting From b57849ca3677b0739ea5d579a57c6f5cd7fed11e Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 5 Oct 2023 23:04:25 +0200 Subject: [PATCH 05/15] Fix review comments --- config_scheme.json | 8 +-- src/json.rs | 135 ++++++++++++++++++++++++++++++++++++++--- src/report/mod.rs | 21 ++++++- src/report/template.rs | 17 +++++- tests/integ/json.yml | 4 +- 5 files changed, 165 insertions(+), 20 deletions(-) diff --git a/config_scheme.json b/config_scheme.json index b4c65e1..23a51f9 100644 --- a/config_scheme.json +++ b/config_scheme.json @@ -151,12 +151,12 @@ "JsonConfig": { "description": "configuration for the json compare module", "type": "object", - "required": [ - "ignore_keys" - ], "properties": { "ignore_keys": { - "type": "array", + "type": [ + "array", + "null" + ], "items": { "type": "string" } diff --git a/src/json.rs b/src/json.rs index 713b470..64f4c2b 100644 --- a/src/json.rs +++ b/src/json.rs @@ -1,6 +1,7 @@ use crate::report::{DiffDetail, Difference}; use crate::Error; use itertools::Itertools; +use regex::Regex; use schemars_derive::JsonSchema; use serde::{Deserialize, Serialize}; use std::path::Path; @@ -9,7 +10,20 @@ use tracing::error; #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] /// configuration for the json compare module pub struct JsonConfig { - ignore_keys: Vec, + ignore_keys: Option>, +} +impl JsonConfig { + pub(crate) fn get_ignore_list(&self) -> Result, regex::Error> { + let exclusion_list: Option, regex::Error>> = self + .ignore_keys + .as_ref() + .map(|v| v.iter().map(|v| Regex::new(v)).collect()); + return if let Some(result) = exclusion_list { + result + } else { + Ok(Vec::new()) + }; + } } pub(crate) fn compare_files>( @@ -22,6 +36,7 @@ pub(crate) fn compare_files>( let nominal = vg_errortools::fat_io_wrap_std(&nominal, &std::fs::read_to_string)?; let actual = vg_errortools::fat_io_wrap_std(&actual, &std::fs::read_to_string)?; + let ignores = config.get_ignore_list()?; let json_diff = json_diff::process::compare_jsons(&nominal, &actual); let json_diff = match json_diff { @@ -38,24 +53,128 @@ pub(crate) fn compare_files>( let filtered_diff: Vec<_> = json_diff .all_diffs() .into_iter() - .filter(|(_d, v)| !config.ignore_keys.contains(v)) + .filter(|(_d, v)| !ignores.iter().any(|excl| excl.is_match(v.get_key()))) .collect(); if !filtered_diff.is_empty() { - let all_diffs_log = filtered_diff + for (d_type, key) in filtered_diff.iter() { + error!("{d_type}: {key}"); + } + let left = filtered_diff + .iter() + .filter_map(|(k, v)| { + if matches!(k, json_diff::enums::DiffType::LeftExtra) { + Some(v.to_string()) + } else { + None + } + }) + .join("\n"); + let right = filtered_diff .iter() - .map(|(d, v)| format!("{d}: {v}")) + .filter_map(|(k, v)| { + if matches!(k, json_diff::enums::DiffType::RightExtra) { + Some(v.to_string()) + } else { + None + } + }) .join("\n"); + let differences = filtered_diff + .iter() + .filter_map(|(k, v)| { + if matches!(k, json_diff::enums::DiffType::Mismatch) { + Some(v.to_string()) + } else { + None + } + }) + .join("\n"); + let root_mismatch = filtered_diff + .iter() + .find(|(k, _v)| matches!(k, json_diff::enums::DiffType::RootMismatch)) + .map(|(_, v)| v.to_string()); + diff.push_detail(DiffDetail::Json { - differences: all_diffs_log, + differences, + left, + right, + root_mismatch, }); diff.error(); } - for (d_type, key) in filtered_diff { - error!("{d_type}: {key}"); + Ok(diff) +} + +#[cfg(test)] +mod test { + use super::*; + + fn trim_split(list: &str) -> Vec<&str> { + list.split("\n").map(|e| e.trim()).collect() + } + + #[test] + fn no_filter() { + let cfg = JsonConfig { + ignore_keys: Some(vec![]), + }; + let result = compare_files( + "tests/integ/data/json/expected/guy.json", + "tests/integ/data/json/actual/guy.json", + &cfg, + ) + .unwrap(); + if let DiffDetail::Json { + differences, + left, + right, + root_mismatch, + } = result.detail.first().unwrap() + { + let differences = trim_split(differences); + assert!(differences.contains(&"car -> [ \"RX7\" :: \"Panda Trueno\" ]")); + assert!(differences.contains(&"age -> [ 21 :: 18 ]")); + assert!(differences.contains(&"name -> [ \"Keisuke\" :: \"Takumi\" ]")); + assert_eq!(differences.len(), 3); + + assert_eq!(left.as_str(), " brothers"); + assert!(right.is_empty()); + assert!(root_mismatch.is_none()); + } else { + panic!("wrong diffdetail"); + } } - Ok(diff) + #[test] + fn filter_works() { + let cfg = JsonConfig { + ignore_keys: Some(vec!["name".to_string(), "brother(s?)".to_string()]), + }; + let result = compare_files( + "tests/integ/data/json/expected/guy.json", + "tests/integ/data/json/actual/guy.json", + &cfg, + ) + .unwrap(); + if let DiffDetail::Json { + differences, + left, + right, + root_mismatch, + } = result.detail.first().unwrap() + { + let differences = trim_split(differences); + assert!(differences.contains(&"car -> [ \"RX7\" :: \"Panda Trueno\" ]")); + assert!(differences.contains(&"age -> [ 21 :: 18 ]")); + assert_eq!(differences.len(), 2); + assert!(right.is_empty()); + assert!(left.is_empty()); + assert!(root_mismatch.is_none()); + } else { + panic!("wrong diffdetail"); + } + } } diff --git a/src/report/mod.rs b/src/report/mod.rs index dbcda29..f974a9c 100644 --- a/src/report/mod.rs +++ b/src/report/mod.rs @@ -148,6 +148,9 @@ pub enum DiffDetail { }, Json { differences: String, + right: String, + left: String, + root_mismatch: Option, }, Properties(MetaDataPropertyDiff), Error(String), @@ -506,7 +509,10 @@ pub fn write_external_detail( pub fn write_json_detail( nominal: impl AsRef, actual: impl AsRef, + left: &str, + right: &str, differences: &str, + root_mismatch: &Option, report_dir: impl AsRef, ) -> Result, Error> { let detail_path = create_detail_folder(report_dir.as_ref())?; @@ -522,6 +528,9 @@ pub fn write_json_detail( ctx.insert("actual", &actual.as_ref().to_string_lossy()); ctx.insert("nominal", &nominal.as_ref().to_string_lossy()); ctx.insert("differences", differences); + ctx.insert("left", left); + ctx.insert("right", right); + ctx.insert("root_mismatch", root_mismatch); let file = fat_io_wrap_std(&detail_file, &File::create)?; debug!("detail html {:?} created", &detail_file); @@ -757,11 +766,16 @@ pub(crate) fn create_html( } } ComparisonMode::Json(_) => { - if let Some(differences) = file + if let Some((differences, left, right, root_mismatch)) = file .detail .iter() .filter_map(|r| match r { - DiffDetail::Json { differences } => Some(differences), + DiffDetail::Json { + left, + differences, + right, + root_mismatch, + } => Some((differences, left, right, root_mismatch)), _ => None, }) .next() @@ -769,7 +783,10 @@ pub(crate) fn create_html( write_json_detail( &file.nominal_file, &file.actual_file, + left, + right, differences, + root_mismatch, &sub_folder, ) .unwrap_or_else(|e| log_detail_html_creation_error(&e)) diff --git a/src/report/template.rs b/src/report/template.rs index cafe67d..73cddd0 100644 --- a/src/report/template.rs +++ b/src/report/template.rs @@ -714,9 +714,12 @@ pub const PLAIN_JSON_DETAIL_TEMPLATE: &str = r###" color: #0d6efdf0; } - .has_error { + .has_right { color:red; } + .has_left { + color:green; + } #compare th { text-align:left; @@ -741,18 +744,26 @@ pub const PLAIN_JSON_DETAIL_TEMPLATE: &str = r###"

Compare Result of {{ actual }} and {{ nominal }}

- +
{{ root_mismatch }}
+ + - + +
Left extra DifferencesRight extra
+ +{{ left }} + {{ differences }} +{{ right }} +
diff --git a/tests/integ/json.yml b/tests/integ/json.yml index b771bfb..1547683 100644 --- a/tests/integ/json.yml +++ b/tests/integ/json.yml @@ -2,6 +2,4 @@ rules: - name: "Compare JSON files" pattern_include: - "**/*.json" - Json: - ignore_keys: - - "" \ No newline at end of file + Json: {} \ No newline at end of file From 3fcfcac081037b470432a30ebc48540e3a3e6a9e Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 5 Oct 2023 23:09:42 +0200 Subject: [PATCH 06/15] Fixup right/left colorization --- src/report/template.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/report/template.rs b/src/report/template.rs index 73cddd0..cdd8225 100644 --- a/src/report/template.rs +++ b/src/report/template.rs @@ -715,10 +715,10 @@ pub const PLAIN_JSON_DETAIL_TEMPLATE: &str = r###" } .has_right { - color:red; + color:green; } .has_left { - color:green; + color:red; } #compare th { From 071e244df3a3d66e938389734f148c26871be844 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 5 Oct 2023 23:12:44 +0200 Subject: [PATCH 07/15] Fix linting --- src/report/template.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/report/template.rs b/src/report/template.rs index cdd8225..de9cd4b 100644 --- a/src/report/template.rs +++ b/src/report/template.rs @@ -1,6 +1,6 @@ pub const INDEX_FILENAME: &str = "index.html"; pub const DETAIL_FILENAME: &str = "detail.html"; -pub const INDEX_TEMPLATE: &str = r###" +pub const INDEX_TEMPLATE: &str = r##" @@ -127,9 +127,9 @@ pub const INDEX_TEMPLATE: &str = r###" -"###; +"##; -pub const PLAIN_TEXT_DETAIL_TEMPLATE: &str = r###" +pub const PLAIN_TEXT_DETAIL_TEMPLATE: &str = r#" @@ -186,9 +186,9 @@ pub const PLAIN_TEXT_DETAIL_TEMPLATE: &str = r###" -"###; +"#; -pub const PLAIN_IMAGE_DETAIL_TEMPLATE: &str = r###" +pub const PLAIN_IMAGE_DETAIL_TEMPLATE: &str = r#" @@ -258,9 +258,9 @@ pub const PLAIN_IMAGE_DETAIL_TEMPLATE: &str = r###" -"###; +"#; -pub const PLAIN_CSV_DETAIL_TEMPLATE: &str = r###" +pub const PLAIN_CSV_DETAIL_TEMPLATE: &str = r#" @@ -398,9 +398,9 @@ pub const PLAIN_CSV_DETAIL_TEMPLATE: &str = r###" -"###; +"#; -pub const PLAIN_PDF_DETAIL_TEMPLATE: &str = r###" +pub const PLAIN_PDF_DETAIL_TEMPLATE: &str = r#" @@ -522,9 +522,9 @@ The extracted exact text can be downloaded here: @@ -600,9 +600,9 @@ pub const ERROR_DETAIL_TEMPLATE: &str = r###" -"###; +"#; -pub const PLAIN_EXTERNAL_DETAIL_TEMPLATE: &str = r###" +pub const PLAIN_EXTERNAL_DETAIL_TEMPLATE: &str = r#" @@ -682,9 +682,9 @@ pub const PLAIN_EXTERNAL_DETAIL_TEMPLATE: &str = r###" -"###; +"#; -pub const PLAIN_JSON_DETAIL_TEMPLATE: &str = r###" +pub const PLAIN_JSON_DETAIL_TEMPLATE: &str = r#" @@ -771,4 +771,4 @@ pub const PLAIN_JSON_DETAIL_TEMPLATE: &str = r###" -"###; +"#; From a6ef2c609cf820e2e9a36e9af53f7ec9be815276 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 5 Oct 2023 23:28:45 +0200 Subject: [PATCH 08/15] Set json-diff to tagged version --- Cargo.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bf8b82b..a0fdce7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,9 +47,7 @@ tempfile = "3.8" fs_extra = "1.3" opener = "0.6" anyhow = "1.0" -json_diff = {git = "https://github.com/ChrisRega/json-diff"} - - +json_diff = {git = "https://github.com/ChrisRega/json-diff", tag = "0.2.0-rc1"} [dev-dependencies] From 31f63114835d60d6cbb3097c932dc3be7d957d62 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 5 Oct 2023 23:31:11 +0200 Subject: [PATCH 09/15] Fix return linting, Add RC1 to indicate pre-release --- Cargo.toml | 2 +- src/json.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a0fdce7..ec1cb62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ description = "A flexible rule-based file and folder comparison tool and crate i repository = "https://github.com/VolumeGraphics/havocompare" homepage = "https://github.com/VolumeGraphics/havocompare" documentation = "https://docs.rs/havocompare" -version = "0.5.0" +version = "0.5.0-RC1" edition = "2021" license = "MIT" authors = ["Volume Graphics GmbH"] diff --git a/src/json.rs b/src/json.rs index 64f4c2b..eaa9bb2 100644 --- a/src/json.rs +++ b/src/json.rs @@ -18,11 +18,11 @@ impl JsonConfig { .ignore_keys .as_ref() .map(|v| v.iter().map(|v| Regex::new(v)).collect()); - return if let Some(result) = exclusion_list { + if let Some(result) = exclusion_list { result } else { Ok(Vec::new()) - }; + } } } From 7819ad501d24126bf8d4a256345f5b458db410ad Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Thu, 5 Oct 2023 23:38:16 +0200 Subject: [PATCH 10/15] Fix README.md --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7f236a6..bff776f 100644 --- a/README.md +++ b/README.md @@ -229,17 +229,18 @@ rules: #### JSON comparison Compares JSON files for different keys in both files and mismatches in values. -ignore_keys elements will be ignored, no substring or regex is supported here. -If you need this, please file a bug. +ignore_keys elements will be ignored, full regex matching on only the key names / paths is supported. +The values are not affected by this. ```yaml rules: - name: "Compare JSON files" pattern_include: - "**/*.json" - Json: + Json: ignore_keys: - - "" + # drop "ignore_this_key" and "ignore_this_keys" with this regex :) + - "ignore_this_key(s?)" ``` From 52c5f976bf181254837288e9e59364957cf19c40 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 6 Oct 2023 21:25:15 +0200 Subject: [PATCH 11/15] integ test for identity serde default --- src/json.rs | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/json.rs b/src/json.rs index eaa9bb2..e4ae1eb 100644 --- a/src/json.rs +++ b/src/json.rs @@ -10,19 +10,12 @@ use tracing::error; #[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] /// configuration for the json compare module pub struct JsonConfig { - ignore_keys: Option>, + #[serde(default)] + ignore_keys: Vec, } impl JsonConfig { pub(crate) fn get_ignore_list(&self) -> Result, regex::Error> { - let exclusion_list: Option, regex::Error>> = self - .ignore_keys - .as_ref() - .map(|v| v.iter().map(|v| Regex::new(v)).collect()); - if let Some(result) = exclusion_list { - result - } else { - Ok(Vec::new()) - } + self.ignore_keys.iter().map(|v| Regex::new(v)).collect() } } @@ -115,11 +108,36 @@ mod test { fn trim_split(list: &str) -> Vec<&str> { list.split("\n").map(|e| e.trim()).collect() } - + #[test] + fn identity_is_empty() { + let cfg = JsonConfig { + ignore_keys: vec![], + }; + let result = compare_files( + "tests/integ/data/json/expected/guy.json", + "tests/integ/data/json/expected/guy.json", + &cfg, + ) + .unwrap(); + if let DiffDetail::Json { + differences, + left, + right, + root_mismatch, + } = result.detail.first().unwrap() + { + assert!(differences.is_empty()); + assert!(left.is_empty()); + assert!(right.is_empty()); + assert!(root_mismatch.is_none()); + } else { + panic!("wrong diffdetail"); + } + } #[test] fn no_filter() { let cfg = JsonConfig { - ignore_keys: Some(vec![]), + ignore_keys: vec![], }; let result = compare_files( "tests/integ/data/json/expected/guy.json", @@ -151,7 +169,7 @@ mod test { #[test] fn filter_works() { let cfg = JsonConfig { - ignore_keys: Some(vec!["name".to_string(), "brother(s?)".to_string()]), + ignore_keys: vec!["name".to_string(), "brother(s?)".to_string()], }; let result = compare_files( "tests/integ/data/json/expected/guy.json", From fedb34d1d760921bd4569a62c32302a793720e08 Mon Sep 17 00:00:00 2001 From: Christopher Regali <60792386+ChrisRega@users.noreply.github.com> Date: Fri, 6 Oct 2023 21:26:32 +0200 Subject: [PATCH 12/15] Update tests/integ.rs Co-authored-by: Alexander Rohde --- tests/integ.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integ.rs b/tests/integ.rs index 0c95158..c403017 100644 --- a/tests/integ.rs +++ b/tests/integ.rs @@ -49,4 +49,14 @@ fn json_test() { report_dir ) .unwrap()); + let report_dir = + tempfile::tempdir().expect("Could not generate temporary directory for report"); + + assert!(compare_folders( + "tests/integ/data/json/expected/", + "tests/integ/data/json/expected/", + "tests/integ/json.yml", + report_dir + ) + .unwrap()); } From 5e51abab39d7488ab7fbc87a4b99feca916c0e9d Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 6 Oct 2023 22:12:30 +0200 Subject: [PATCH 13/15] Fix empty results in json_diff --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ec1cb62..3a4045d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ tempfile = "3.8" fs_extra = "1.3" opener = "0.6" anyhow = "1.0" -json_diff = {git = "https://github.com/ChrisRega/json-diff", tag = "0.2.0-rc1"} +json_diff = {git = "https://github.com/ChrisRega/json-diff", tag = "0.2.0-rc2"} [dev-dependencies] From c79da4e858e1cdf0241465bb6f49d466a41024e0 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 6 Oct 2023 22:21:15 +0200 Subject: [PATCH 14/15] Remove broken unit test in favor of integ test --- src/json.rs | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/json.rs b/src/json.rs index e4ae1eb..9ed2681 100644 --- a/src/json.rs +++ b/src/json.rs @@ -108,32 +108,7 @@ mod test { fn trim_split(list: &str) -> Vec<&str> { list.split("\n").map(|e| e.trim()).collect() } - #[test] - fn identity_is_empty() { - let cfg = JsonConfig { - ignore_keys: vec![], - }; - let result = compare_files( - "tests/integ/data/json/expected/guy.json", - "tests/integ/data/json/expected/guy.json", - &cfg, - ) - .unwrap(); - if let DiffDetail::Json { - differences, - left, - right, - root_mismatch, - } = result.detail.first().unwrap() - { - assert!(differences.is_empty()); - assert!(left.is_empty()); - assert!(right.is_empty()); - assert!(root_mismatch.is_none()); - } else { - panic!("wrong diffdetail"); - } - } + #[test] fn no_filter() { let cfg = JsonConfig { From 5b5861e5f2eef77c0cdca247f7793b46bf08e549 Mon Sep 17 00:00:00 2001 From: Christopher Regali Date: Fri, 6 Oct 2023 22:29:24 +0200 Subject: [PATCH 15/15] Update coverallsaction --- .github/workflows/coverage.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 9124a9e..ec1fb1f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install latest nightly uses: actions-rs/toolchain@v1 with: @@ -22,7 +22,7 @@ jobs: - name: install grcov run: cargo install grcov - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -37,8 +37,8 @@ jobs: grcov . -s . --binary-path ./target/debug/ -t lcov --llvm --branch --ignore-not-existing --ignore="/*" --ignore="target/*" --ignore="tests/*" -o lcov.info - name: Push grcov results to Coveralls via GitHub Action - uses: coverallsapp/github-action@v1.0.1 + uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} - path-to-lcov: "lcov.info" + file: "lcov.info"