From 0e6bed9a328db00071725ba5c668be1870893992 Mon Sep 17 00:00:00 2001 From: Elizabeth Harper Date: Thu, 18 Oct 2018 10:57:02 -0400 Subject: [PATCH] Fixes and improvements: * Validation errors are now aggregated into an output list. Resolves #4 * Output directory is now optional. The tool will only perform validations in this case. Resolves #1 * Path trim error fixed. Resolves #2 * Exception stack traces are now ommitted for validation errors. Resolves #3 --- .gitignore | 1 + readme.adoc | 11 +-- source/app.d | 36 ++++++++-- source/lib/deref.d | 103 ++++++++++++++++++++------- source/lib/err/BadJsonException.d | 13 ++++ source/lib/err/InvalidRefException.d | 14 ++++ source/lib/err/NotFoundException.d | 13 ++++ source/lib/err/ValidationException.d | 9 +++ source/lib/err/package.d | 6 ++ 9 files changed, 170 insertions(+), 36 deletions(-) create mode 100644 source/lib/err/BadJsonException.d create mode 100644 source/lib/err/InvalidRefException.d create mode 100644 source/lib/err/NotFoundException.d create mode 100644 source/lib/err/ValidationException.d create mode 100644 source/lib/err/package.d diff --git a/.gitignore b/.gitignore index f2057ac..4141c4c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ jss-deref-test-* *.o *.obj *.lst +js-deref diff --git a/readme.adoc b/readme.adoc index ca82a21..3ecf02e 100644 --- a/readme.adoc +++ b/readme.adoc @@ -5,12 +5,15 @@ image:https://img.shields.io/github/release/EuPathDB-Infra/js-deref.svg?style=flat-square[GitHub tag,link=https://github.com/EuPathDB-Infra/js-deref/releases/latest] image:https://img.shields.io/travis/EuPathDB-Infra/js-deref/master.svg?style=flat-square[Travis (.org) branch,link=https://travis-ci.org/EuPathDB-Infra/js-deref] - Recursively resolves `"$ref"` properties and appends the external schema content to the root `"definitions"` object. -Reads entire directory trees and copies the input files to -the output location. +If an output directory is provided, the tool writes the +dereferenced json files to the output directory, mirroring +the structure of the input directory. + +If no output directory is provided, the tool simply performs +the validation checks. WARNING: This utility is not presently suited for general use as a json schema ref resolver. It is tailored to the @@ -21,7 +24,7 @@ root of a java project's resource directory. [source, bash session] ---- -$ js-deref +$ js-deref [output_dir] ---- == Rule checks diff --git a/source/app.d b/source/app.d index 7e8da49..aa22e07 100644 --- a/source/app.d +++ b/source/app.d @@ -1,20 +1,46 @@ -import std.file: getcwd; +import std.file: getcwd, exists; import std.path: buildPath; -import std.stdio: writeln; +import std.stdio: writefln, writeln, stderr; import lib.deref; +import lib.err.valid: ValidationException; int main(string[] args) { if (args.length < 2) { writeln(`js-deref usage: - js-deref `); + js-deref [output-dir]`); return 1; } const string cwd = getcwd(); - new Dereferencer(cwd.buildPath(args[1]), cwd.buildPath(args[2])).run(); - return 0; + Dereferencer deref; + const string inPath = cwd.buildPath(args[1]).trailingSlash(); + + if (!inPath.exists()) { + writefln("Input path does not exist: %s", inPath); + return 1; + } + + if (args.length == 2) { + deref = new Dereferencer(inPath); + } else { + const string outPath = cwd.buildPath(args[2]).trailingSlash(); + deref = new Dereferencer(inPath, outPath); + } + + try { + return deref.run(); + } catch (ValidationException e) { + stderr.writeln(e.msg); + return 1; + } +} + +//TODO: Rehome us + +private string trailingSlash(const string path) { + return path[$-1] == '/' ? path : path ~ '/'; } diff --git a/source/lib/deref.d b/source/lib/deref.d index e78dbd0..c249f1f 100644 --- a/source/lib/deref.d +++ b/source/lib/deref.d @@ -1,6 +1,6 @@ module lib.deref; -import std.stdio: writefln, toFile; +import std.stdio: writefln, writeln, toFile; import std.file: dirEntries, SpanMode, readText, mkdirRecurse; import std.path: buildNormalizedPath, dirName, baseName, buildPath; import std.json: JSONValue, parseJSON, JSONException, JSONType; @@ -8,22 +8,13 @@ import std.string: indexOf, split, format; import lib.schema; import lib.json_schema; +import lib.err; debug(deref) import lib.bug.pdb: printDebug; private const string INCLUDES_DIR = "includes"; private const string SCHEMA_FILTER = "*.json"; -private const string ERR_ILLEGAL_REF = "References must be absolute paths to " - ~ `the root of the resource directory. Example '/schema/wdk/...'. - Invalid Ref: %s - Schema File: %s`; - -private const string ERR_OUT_OF_SCOPE = "References not found in the resource " - ~ `directory. - Invalid Ref: %s - Schema File: %s`; - private const string ERR_JSON_PARSE = `Failed to parse JSON file contents. Schema File: %s`; @@ -33,6 +24,8 @@ class Dereferencer { private const string inDir; + private const bool writeOutput; + private const string outDir; /** @@ -43,32 +36,70 @@ class Dereferencer { debug(deref) printDebug(["inDir": inDir, "outDir": outDir]); this.inDir = inDir; this.outDir = outDir; + this.writeOutput = true; + } + + /** + * @param inDir input schema home + * @param outDir schema output directory + */ + this(const string inDir) { + debug(deref) printDebug(["inDir": inDir]); + this.inDir = inDir; + this.outDir = ""; + this.writeOutput = false; } /** * */ - public void run() { + public int run() { debug(deref) printDebug([]); - scanDir(); + if (!scanDir()) + return 1; - foreach(const string path, Schema s; all) { - if (!s.hasRef()) - continue; + if (!seekAll(this.all)) + return 1; - seek(s, s.getValue()); - } + if(!writeOutput) + return 0; foreach(Schema s; all) { s.resolveDefs(); writeOut(s); } + + return 0; + } + + private bool seekAll(ref Schema[string] all) { + string[] errors; + + foreach(const string path, Schema s; all) { + if (!s.hasRef()) + continue; + + try { + seek(s, s.getValue()); + } catch (ValidationException e) { + errors ~= e.msg; + } + } + + if (errors.length == 0) + return true; + + writefln("Error: validation failed for %d schema files.\n", errors.length); + foreach(const string e; errors) + writefln("%s\n", e); + + return false; } private void writeOut(Schema s) { debug(deref) printDebug(["s": &s]); - + const string outPath = buildPath(outDir, s.getLocalDir()); if (baseName(outPath) == INCLUDES_DIR) @@ -127,7 +158,7 @@ class Dereferencer { */ void seekArray(Schema s, JSONValue* value) { debug(deref) printDebug(["s": &s, "value": value]); - + foreach(ref JSONValue cur; value.array) seek(s, &cur); } @@ -140,7 +171,7 @@ class Dereferencer { debug(deref) printDebug(["path": &path, "s": &s]); if(path[0] != '/') - throw new Exception(format(ERR_ILLEGAL_REF, path, s.getPath())); + throw new InvalidRefException(path, s.getPath()); return localResolve(path, s); } @@ -155,7 +186,7 @@ class Dereferencer { Schema* found = (normal in all); if (found is null) { - throw new Exception(format(ERR_OUT_OF_SCOPE, path, s.getPath())); + throw new NotFoundException(path, s.getPath()); } return *found; @@ -165,11 +196,26 @@ class Dereferencer { * Scan given directory for entries matching SCHEMA_FILTER * and pass them to processFile */ - private void scanDir() { + private bool scanDir() { debug(deref) printDebug([]); + string[] errors; + + foreach (string file; dirEntries(inDir, SCHEMA_FILTER, SpanMode.breadth)) { + try { + processFile(file); + } catch (ValidationException e) { + errors ~= e.msg; + } + } - foreach (string file; dirEntries(inDir, SCHEMA_FILTER, SpanMode.breadth)) - processFile(file); + if (errors.length > 0) { + writefln("Error: %d files could not be read due to JSON syntax errors", errors.length); + foreach (const string e; errors) { + writeln(e); + } + return false; + } + return true; } /** @@ -180,15 +226,18 @@ class Dereferencer { debug(deref) printDebug(["path": &path]); const string body = readText(path); + const string shortPath = dirName(path[inDir.length..$]); try { - all[path] = new Schema(baseName(path), dirName(path[inDir.length + 1..$]), checkRefs(body), parseJSON(body)); + all[path] = new Schema(baseName(path), shortPath, checkRefs(body), parseJSON(body)); } catch(JSONException e) { - throw new Exception(format(ERR_JSON_PARSE, path), e); + throw new BadJsonException(shortPath, e); } } } + + private string defPath(const string next) { debug(deref) printDebug(["next": &next]); diff --git a/source/lib/err/BadJsonException.d b/source/lib/err/BadJsonException.d new file mode 100644 index 0000000..736ac38 --- /dev/null +++ b/source/lib/err/BadJsonException.d @@ -0,0 +1,13 @@ +module lib.err.json; + +import std.json: JSONException; +import std.string: format; + +import lib.err.valid : ValidationException; + +public class BadJsonException: ValidationException { + public this(const string file, const JSONException e) { + super(format(`%s: + %s`, file, e.msg)); + } +} diff --git a/source/lib/err/InvalidRefException.d b/source/lib/err/InvalidRefException.d new file mode 100644 index 0000000..30e08e8 --- /dev/null +++ b/source/lib/err/InvalidRefException.d @@ -0,0 +1,14 @@ +module lib.err.iref; + +import std.string: format; + +import lib.err.valid: ValidationException; + +public class InvalidRefException : ValidationException { + this(const string path, const string schema) { + super(format("References must be absolute paths to " + ~ `the root of the resource directory. Example '/schema/wdk/...'. + Invalid Ref: %s + Schema File: %s`, path, schema)); + } +} diff --git a/source/lib/err/NotFoundException.d b/source/lib/err/NotFoundException.d new file mode 100644 index 0000000..155360d --- /dev/null +++ b/source/lib/err/NotFoundException.d @@ -0,0 +1,13 @@ +module lib.err.nf; + +import std.string: format; + +import lib.err.valid: ValidationException; + +public class NotFoundException : ValidationException { + this(const string path, const string schema) { + super(format(`References not found in the resource directory. + Invalid Ref: %s + Schema File: %s`, path, schema)); + } +} \ No newline at end of file diff --git a/source/lib/err/ValidationException.d b/source/lib/err/ValidationException.d new file mode 100644 index 0000000..64d72eb --- /dev/null +++ b/source/lib/err/ValidationException.d @@ -0,0 +1,9 @@ +module lib.err.valid; + +import std.exception; + +public class ValidationException : Exception { + public this(const string message) { + super(message); + } +} diff --git a/source/lib/err/package.d b/source/lib/err/package.d new file mode 100644 index 0000000..4deedf7 --- /dev/null +++ b/source/lib/err/package.d @@ -0,0 +1,6 @@ +module lib.err; + +public import lib.err.nf; +public import lib.err.iref; +public import lib.err.valid; +public import lib.err.json;