Add json diff

.github/workflows/coverage.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Install latest nightly
       uses: actions-rs/toolchain@v1
       with:
@@ -22,7 +22,7 @@ jobs:
     - name: install grcov
       run: cargo install grcov
 
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
@@ -37,8 +37,8 @@ jobs:
          grcov . -s . --binary-path ./target/debug/ -t lcov --llvm --branch --ignore-not-existing --ignore="/*" --ignore="target/*" --ignore="tests/*"  -o lcov.info
 
     - name: Push grcov results to Coveralls via GitHub Action
-      uses: coverallsapp/github-action@v1.0.1
+      uses: coverallsapp/github-action@v2
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
-        path-to-lcov: "lcov.info"
+        file: "lcov.info"
 

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "havocompare"
-description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, text files, pdf-texts and images."
+description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, JSON, text files, pdf-texts and images."
 repository = "https://github.com/VolumeGraphics/havocompare"
 homepage = "https://github.com/VolumeGraphics/havocompare"
 documentation = "https://docs.rs/havocompare"
-version = "0.4.0"
+version = "0.5.0-RC1"
 edition = "2021"
 license = "MIT"
 authors = ["Volume Graphics GmbH"]
@@ -18,7 +18,7 @@ path = "src/print_args.rs"
 
 
 [dependencies]
-clap = {version= "4.3", features=["derive"]}
+clap = {version= "4.4", features=["derive"]}
 chrono = "0.4"
 serde = "1.0"
 serde_yaml = "0.9"
@@ -27,7 +27,7 @@ schemars_derive = "0.8"
 thiserror = "1.0"
 regex = "1.8"
 image = "0.24"
-image-compare = "0.3.0"
+image-compare = "0.3.1"
 tracing = "0.1"
 tracing-subscriber = "0.3"
 serde_json = "1.0"
@@ -39,14 +39,16 @@ tera = "1.19"
 sha2 = "0.10"
 data-encoding = "2.4"
 permutation = "0.4"
-pdf-extract = "0.6"
+pdf-extract = "0.7"
 vg_errortools = "0.1"
-rayon = "1.7.0"
+rayon = "1.8.0"
 enable-ansi-support = "0.2"
-tempfile = "3.6"
+tempfile = "3.8"
 fs_extra = "1.3"
 opener = "0.6"
 anyhow = "1.0"
+json_diff = {git = "https://github.com/ChrisRega/json-diff", tag = "0.2.0-rc2"}
+
 
 [dev-dependencies]
 env_logger = "0.10"

README.md

@@ -5,6 +5,11 @@
 [![Coverage Status](https://coveralls.io/repos/github/VolumeGraphics/havocompare/badge.svg?branch=main)](https://coveralls.io/github/VolumeGraphics/havocompare?branch=main)
 [![License](https://img.shields.io/badge/license-MIT-blue?style=flat)](LICENSE)
 
+## Contributors:
+<a href="https://github.com/VolumeGraphics/havocompare/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=VolumeGraphics/havocompare"  alt="Contributors"/>
+</a>
+
 ## Quickstart
 
 ### 0. Install havocompare
@@ -221,8 +226,29 @@ rules:
         - "--only-images"
 ```
 
+
+#### JSON comparison
+Compares JSON files for different keys in both files and mismatches in values.
+ignore_keys elements will be ignored, full regex matching on only the key names / paths is supported.
+The values are not affected by this.
+
+```yaml
+rules:
+- name: "Compare JSON files"
+  pattern_include:
+  - "**/*.json"
+  Json: 
+    ignore_keys:
+      # drop "ignore_this_key" and "ignore_this_keys" with this regex :)
+      - "ignore_this_key(s?)"
+```
+
+
 ## Changelog
 
+### 0.5.0
+- Add basic JSON checking
+
 ### 0.4.0
 - Separate reporting logic from comparison logic
 - Implement a machine-readable JSON reporting

config_scheme.json

@@ -148,6 +148,21 @@
         }
       }
     },
+    "JsonConfig": {
+      "description": "configuration for the json compare module",
+      "type": "object",
+      "properties": {
+        "ignore_keys": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "Mode": {
       "description": "comparison mode for csv cells",
       "oneOf": [
@@ -457,6 +472,19 @@
           },
           "additionalProperties": false
         },
+        {
+          "description": "Compare JSON files",
+          "type": "object",
+          "required": [
+            "Json"
+          ],
+          "properties": {
+            "Json": {
+              "$ref": "#/definitions/JsonConfig"
+            }
+          },
+          "additionalProperties": false
+        },
         {
           "description": "Run external comparison executable",
           "type": "object",

src/json.rs

@@ -0,0 +1,173 @@
+use crate::report::{DiffDetail, Difference};
+use crate::Error;
+use itertools::Itertools;
+use regex::Regex;
+use schemars_derive::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+use tracing::error;
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+/// configuration for the json compare module
+pub struct JsonConfig {
+    #[serde(default)]
+    ignore_keys: Vec<String>,
+}
+impl JsonConfig {
+    pub(crate) fn get_ignore_list(&self) -> Result<Vec<Regex>, regex::Error> {
+        self.ignore_keys.iter().map(|v| Regex::new(v)).collect()
+    }
+}
+
+pub(crate) fn compare_files<P: AsRef<Path>>(
+    nominal: P,
+    actual: P,
+    config: &JsonConfig,
+) -> Result<Difference, Error> {
+    let mut diff = Difference::new_for_file(&nominal, &actual);
+    let compared_file_name = nominal.as_ref().to_string_lossy().into_owned();
+
+    let nominal = vg_errortools::fat_io_wrap_std(&nominal, &std::fs::read_to_string)?;
+    let actual = vg_errortools::fat_io_wrap_std(&actual, &std::fs::read_to_string)?;
+    let ignores = config.get_ignore_list()?;
+
+    let json_diff = json_diff::process::compare_jsons(&nominal, &actual);
+    let json_diff = match json_diff {
+        Ok(diff) => diff,
+        Err(e) => {
+            let error_message =
+                format!("JSON deserialization failed for {compared_file_name} (error: {e})");
+            error!("{}", error_message);
+            diff.push_detail(DiffDetail::Error(error_message));
+            diff.error();
+            return Ok(diff);
+        }
+    };
+    let filtered_diff: Vec<_> = json_diff
+        .all_diffs()
+        .into_iter()
+        .filter(|(_d, v)| !ignores.iter().any(|excl| excl.is_match(v.get_key())))
+        .collect();
+
+    if !filtered_diff.is_empty() {
+        for (d_type, key) in filtered_diff.iter() {
+            error!("{d_type}: {key}");
+        }
+        let left = filtered_diff
+            .iter()
+            .filter_map(|(k, v)| {
+                if matches!(k, json_diff::enums::DiffType::LeftExtra) {
+                    Some(v.to_string())
+                } else {
+                    None
+                }
+            })
+            .join("\n");
+        let right = filtered_diff
+            .iter()
+            .filter_map(|(k, v)| {
+                if matches!(k, json_diff::enums::DiffType::RightExtra) {
+                    Some(v.to_string())
+                } else {
+                    None
+                }
+            })
+            .join("\n");
+        let differences = filtered_diff
+            .iter()
+            .filter_map(|(k, v)| {
+                if matches!(k, json_diff::enums::DiffType::Mismatch) {
+                    Some(v.to_string())
+                } else {
+                    None
+                }
+            })
+            .join("\n");
+        let root_mismatch = filtered_diff
+            .iter()
+            .find(|(k, _v)| matches!(k, json_diff::enums::DiffType::RootMismatch))
+            .map(|(_, v)| v.to_string());
+
+        diff.push_detail(DiffDetail::Json {
+            differences,
+            left,
+            right,
+            root_mismatch,
+        });
+
+        diff.error();
+    }
+
+    Ok(diff)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn trim_split(list: &str) -> Vec<&str> {
+        list.split("\n").map(|e| e.trim()).collect()
+    }
+
+    #[test]
+    fn no_filter() {
+        let cfg = JsonConfig {
+            ignore_keys: vec![],
+        };
+        let result = compare_files(
+            "tests/integ/data/json/expected/guy.json",
+            "tests/integ/data/json/actual/guy.json",
+            &cfg,
+        )
+        .unwrap();
+        if let DiffDetail::Json {
+            differences,
+            left,
+            right,
+            root_mismatch,
+        } = result.detail.first().unwrap()
+        {
+            let differences = trim_split(differences);
+            assert!(differences.contains(&"car -> [ \"RX7\" :: \"Panda Trueno\" ]"));
+            assert!(differences.contains(&"age -> [ 21 :: 18 ]"));
+            assert!(differences.contains(&"name -> [ \"Keisuke\" :: \"Takumi\" ]"));
+            assert_eq!(differences.len(), 3);
+
+            assert_eq!(left.as_str(), " brothers");
+            assert!(right.is_empty());
+            assert!(root_mismatch.is_none());
+        } else {
+            panic!("wrong diffdetail");
+        }
+    }
+
+    #[test]
+    fn filter_works() {
+        let cfg = JsonConfig {
+            ignore_keys: vec!["name".to_string(), "brother(s?)".to_string()],
+        };
+        let result = compare_files(
+            "tests/integ/data/json/expected/guy.json",
+            "tests/integ/data/json/actual/guy.json",
+            &cfg,
+        )
+        .unwrap();
+        if let DiffDetail::Json {
+            differences,
+            left,
+            right,
+            root_mismatch,
+        } = result.detail.first().unwrap()
+        {
+            let differences = trim_split(differences);
+            assert!(differences.contains(&"car -> [ \"RX7\" :: \"Panda Trueno\" ]"));
+            assert!(differences.contains(&"age -> [ 21 :: 18 ]"));
+            assert_eq!(differences.len(), 2);
+            assert!(right.is_empty());
+            assert!(left.is_empty());
+            assert!(root_mismatch.is_none());
+        } else {
+            panic!("wrong diffdetail");
+        }
+    }
+}

src/lib.rs

@@ -24,6 +24,9 @@ mod pdf;
 mod properties;
 mod report;
 
+mod json;
+pub use crate::json::JsonConfig;
+
 use crate::external::ExternalConfig;
 pub use crate::html::HTMLCompareConfig;
 use crate::properties::PropertiesConfig;
@@ -93,6 +96,9 @@ pub enum ComparisonMode {
     /// Compare file-properties
     FileProperties(PropertiesConfig),
 
+    /// Compare JSON files
+    Json(JsonConfig),
+
     /// Run external comparison executable
     External(ExternalConfig),
 }
@@ -194,6 +200,9 @@ fn process_file(nominal: impl AsRef<Path>, actual: impl AsRef<Path>, rule: &Rule
                 external::compare_files(nominal.as_ref(), actual.as_ref(), conf)
                     .map_err(|e| e.into())
             }
+            ComparisonMode::Json(conf) => {
+                json::compare_files(nominal.as_ref(), actual.as_ref(), conf).map_err(|e| e.into())
+            }
         }
     };
     let compare_result = match compare_result {