diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 9124a9e..ec1fb1f 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -8,7 +8,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Install latest nightly
uses: actions-rs/toolchain@v1
with:
@@ -22,7 +22,7 @@ jobs:
- name: install grcov
run: cargo install grcov
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
with:
fetch-depth: 0
@@ -37,8 +37,8 @@ jobs:
grcov . -s . --binary-path ./target/debug/ -t lcov --llvm --branch --ignore-not-existing --ignore="/*" --ignore="target/*" --ignore="tests/*" -o lcov.info
- name: Push grcov results to Coveralls via GitHub Action
- uses: coverallsapp/github-action@v1.0.1
+ uses: coverallsapp/github-action@v2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- path-to-lcov: "lcov.info"
+ file: "lcov.info"
diff --git a/Cargo.toml b/Cargo.toml
index f637308..3a4045d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,10 +1,10 @@
[package]
name = "havocompare"
-description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, text files, pdf-texts and images."
+description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, JSON, text files, pdf-texts and images."
repository = "https://github.com/VolumeGraphics/havocompare"
homepage = "https://github.com/VolumeGraphics/havocompare"
documentation = "https://docs.rs/havocompare"
-version = "0.4.0"
+version = "0.5.0-RC1"
edition = "2021"
license = "MIT"
authors = ["Volume Graphics GmbH"]
@@ -18,7 +18,7 @@ path = "src/print_args.rs"
[dependencies]
-clap = {version= "4.3", features=["derive"]}
+clap = {version= "4.4", features=["derive"]}
chrono = "0.4"
serde = "1.0"
serde_yaml = "0.9"
@@ -27,7 +27,7 @@ schemars_derive = "0.8"
thiserror = "1.0"
regex = "1.8"
image = "0.24"
-image-compare = "0.3.0"
+image-compare = "0.3.1"
tracing = "0.1"
tracing-subscriber = "0.3"
serde_json = "1.0"
@@ -39,14 +39,16 @@ tera = "1.19"
sha2 = "0.10"
data-encoding = "2.4"
permutation = "0.4"
-pdf-extract = "0.6"
+pdf-extract = "0.7"
vg_errortools = "0.1"
-rayon = "1.7.0"
+rayon = "1.8.0"
enable-ansi-support = "0.2"
-tempfile = "3.6"
+tempfile = "3.8"
fs_extra = "1.3"
opener = "0.6"
anyhow = "1.0"
+json_diff = {git = "https://github.com/ChrisRega/json-diff", tag = "0.2.0-rc2"}
+
[dev-dependencies]
env_logger = "0.10"
diff --git a/README.md b/README.md
index a885702..bff776f 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,11 @@
[![Coverage Status](https://coveralls.io/repos/github/VolumeGraphics/havocompare/badge.svg?branch=main)](https://coveralls.io/github/VolumeGraphics/havocompare?branch=main)
[![License](https://img.shields.io/badge/license-MIT-blue?style=flat)](LICENSE)
+## Contributors:
+
+
+
+
## Quickstart
### 0. Install havocompare
@@ -221,8 +226,29 @@ rules:
- "--only-images"
```
+
+#### JSON comparison
+Compares JSON files for different keys in both files and mismatches in values.
+ignore_keys elements will be ignored, full regex matching on only the key names / paths is supported.
+The values are not affected by this.
+
+```yaml
+rules:
+- name: "Compare JSON files"
+ pattern_include:
+ - "**/*.json"
+ Json:
+ ignore_keys:
+ # drop "ignore_this_key" and "ignore_this_keys" with this regex :)
+ - "ignore_this_key(s?)"
+```
+
+
## Changelog
+### 0.5.0
+- Add basic JSON checking
+
### 0.4.0
- Separate reporting logic from comparison logic
- Implement a machine-readable JSON reporting
diff --git a/config_scheme.json b/config_scheme.json
index 9ad2606..23a51f9 100644
--- a/config_scheme.json
+++ b/config_scheme.json
@@ -148,6 +148,21 @@
}
}
},
+ "JsonConfig": {
+ "description": "configuration for the json compare module",
+ "type": "object",
+ "properties": {
+ "ignore_keys": {
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
"Mode": {
"description": "comparison mode for csv cells",
"oneOf": [
@@ -457,6 +472,19 @@
},
"additionalProperties": false
},
+ {
+ "description": "Compare JSON files",
+ "type": "object",
+ "required": [
+ "Json"
+ ],
+ "properties": {
+ "Json": {
+ "$ref": "#/definitions/JsonConfig"
+ }
+ },
+ "additionalProperties": false
+ },
{
"description": "Run external comparison executable",
"type": "object",
diff --git a/src/json.rs b/src/json.rs
new file mode 100644
index 0000000..9ed2681
--- /dev/null
+++ b/src/json.rs
@@ -0,0 +1,173 @@
+use crate::report::{DiffDetail, Difference};
+use crate::Error;
+use itertools::Itertools;
+use regex::Regex;
+use schemars_derive::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+use tracing::error;
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+/// configuration for the json compare module
+pub struct JsonConfig {
+ #[serde(default)]
+ ignore_keys: Vec,
+}
+impl JsonConfig {
+ pub(crate) fn get_ignore_list(&self) -> Result, regex::Error> {
+ self.ignore_keys.iter().map(|v| Regex::new(v)).collect()
+ }
+}
+
+pub(crate) fn compare_files>(
+ nominal: P,
+ actual: P,
+ config: &JsonConfig,
+) -> Result {
+ let mut diff = Difference::new_for_file(&nominal, &actual);
+ let compared_file_name = nominal.as_ref().to_string_lossy().into_owned();
+
+ let nominal = vg_errortools::fat_io_wrap_std(&nominal, &std::fs::read_to_string)?;
+ let actual = vg_errortools::fat_io_wrap_std(&actual, &std::fs::read_to_string)?;
+ let ignores = config.get_ignore_list()?;
+
+ let json_diff = json_diff::process::compare_jsons(&nominal, &actual);
+ let json_diff = match json_diff {
+ Ok(diff) => diff,
+ Err(e) => {
+ let error_message =
+ format!("JSON deserialization failed for {compared_file_name} (error: {e})");
+ error!("{}", error_message);
+ diff.push_detail(DiffDetail::Error(error_message));
+ diff.error();
+ return Ok(diff);
+ }
+ };
+ let filtered_diff: Vec<_> = json_diff
+ .all_diffs()
+ .into_iter()
+ .filter(|(_d, v)| !ignores.iter().any(|excl| excl.is_match(v.get_key())))
+ .collect();
+
+ if !filtered_diff.is_empty() {
+ for (d_type, key) in filtered_diff.iter() {
+ error!("{d_type}: {key}");
+ }
+ let left = filtered_diff
+ .iter()
+ .filter_map(|(k, v)| {
+ if matches!(k, json_diff::enums::DiffType::LeftExtra) {
+ Some(v.to_string())
+ } else {
+ None
+ }
+ })
+ .join("\n");
+ let right = filtered_diff
+ .iter()
+ .filter_map(|(k, v)| {
+ if matches!(k, json_diff::enums::DiffType::RightExtra) {
+ Some(v.to_string())
+ } else {
+ None
+ }
+ })
+ .join("\n");
+ let differences = filtered_diff
+ .iter()
+ .filter_map(|(k, v)| {
+ if matches!(k, json_diff::enums::DiffType::Mismatch) {
+ Some(v.to_string())
+ } else {
+ None
+ }
+ })
+ .join("\n");
+ let root_mismatch = filtered_diff
+ .iter()
+ .find(|(k, _v)| matches!(k, json_diff::enums::DiffType::RootMismatch))
+ .map(|(_, v)| v.to_string());
+
+ diff.push_detail(DiffDetail::Json {
+ differences,
+ left,
+ right,
+ root_mismatch,
+ });
+
+ diff.error();
+ }
+
+ Ok(diff)
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ fn trim_split(list: &str) -> Vec<&str> {
+ list.split("\n").map(|e| e.trim()).collect()
+ }
+
+ #[test]
+ fn no_filter() {
+ let cfg = JsonConfig {
+ ignore_keys: vec![],
+ };
+ let result = compare_files(
+ "tests/integ/data/json/expected/guy.json",
+ "tests/integ/data/json/actual/guy.json",
+ &cfg,
+ )
+ .unwrap();
+ if let DiffDetail::Json {
+ differences,
+ left,
+ right,
+ root_mismatch,
+ } = result.detail.first().unwrap()
+ {
+ let differences = trim_split(differences);
+ assert!(differences.contains(&"car -> [ \"RX7\" :: \"Panda Trueno\" ]"));
+ assert!(differences.contains(&"age -> [ 21 :: 18 ]"));
+ assert!(differences.contains(&"name -> [ \"Keisuke\" :: \"Takumi\" ]"));
+ assert_eq!(differences.len(), 3);
+
+ assert_eq!(left.as_str(), " brothers");
+ assert!(right.is_empty());
+ assert!(root_mismatch.is_none());
+ } else {
+ panic!("wrong diffdetail");
+ }
+ }
+
+ #[test]
+ fn filter_works() {
+ let cfg = JsonConfig {
+ ignore_keys: vec!["name".to_string(), "brother(s?)".to_string()],
+ };
+ let result = compare_files(
+ "tests/integ/data/json/expected/guy.json",
+ "tests/integ/data/json/actual/guy.json",
+ &cfg,
+ )
+ .unwrap();
+ if let DiffDetail::Json {
+ differences,
+ left,
+ right,
+ root_mismatch,
+ } = result.detail.first().unwrap()
+ {
+ let differences = trim_split(differences);
+ assert!(differences.contains(&"car -> [ \"RX7\" :: \"Panda Trueno\" ]"));
+ assert!(differences.contains(&"age -> [ 21 :: 18 ]"));
+ assert_eq!(differences.len(), 2);
+ assert!(right.is_empty());
+ assert!(left.is_empty());
+ assert!(root_mismatch.is_none());
+ } else {
+ panic!("wrong diffdetail");
+ }
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index da5f668..4a317d7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -24,6 +24,9 @@ mod pdf;
mod properties;
mod report;
+mod json;
+pub use crate::json::JsonConfig;
+
use crate::external::ExternalConfig;
pub use crate::html::HTMLCompareConfig;
use crate::properties::PropertiesConfig;
@@ -93,6 +96,9 @@ pub enum ComparisonMode {
/// Compare file-properties
FileProperties(PropertiesConfig),
+ /// Compare JSON files
+ Json(JsonConfig),
+
/// Run external comparison executable
External(ExternalConfig),
}
@@ -194,6 +200,9 @@ fn process_file(nominal: impl AsRef, actual: impl AsRef, rule: &Rule
external::compare_files(nominal.as_ref(), actual.as_ref(), conf)
.map_err(|e| e.into())
}
+ ComparisonMode::Json(conf) => {
+ json::compare_files(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into())
+ }
}
};
let compare_result = match compare_result {
diff --git a/src/report/mod.rs b/src/report/mod.rs
index 5a2af8a..f974a9c 100644
--- a/src/report/mod.rs
+++ b/src/report/mod.rs
@@ -119,7 +119,7 @@ impl Difference {
return false;
}
self.is_error |= other.is_error;
- self.detail.extend(other.detail.into_iter());
+ self.detail.extend(other.detail);
true
}
}
@@ -146,6 +146,12 @@ pub enum DiffDetail {
stdout: String,
stderr: String,
},
+ Json {
+ differences: String,
+ right: String,
+ left: String,
+ root_mismatch: Option,
+ },
Properties(MetaDataPropertyDiff),
Error(String),
}
@@ -265,7 +271,7 @@ pub(crate) fn write_csv_detail(
let columns: Vec = n
.into_iter()
- .zip(a.into_iter())
+ .zip(a)
.enumerate()
.map(|(col, (n, a))| {
let current_pos = Position { col, row };
@@ -500,6 +506,40 @@ pub fn write_external_detail(
Ok(Some(detail_path))
}
+pub fn write_json_detail(
+ nominal: impl AsRef,
+ actual: impl AsRef,
+ left: &str,
+ right: &str,
+ differences: &str,
+ root_mismatch: &Option,
+ report_dir: impl AsRef,
+) -> Result