feature(AP) Add AP nodes for clauses and variables

Signed-off-by: Jerome Simeon <[email protected]>

src/Commands.js

@@ -18,6 +18,7 @@ const fs = require('fs');
 const Logger = require('./Logger');
 const CommonmarkParser = require('./CommonmarkParser');
 const CommonmarkToString = require('./CommonmarkToString');
+const CommonmarkToAP = require('./CommonmarkToAP');
 
 /**
  * Utility class that implements the commands exposed by the CLI.
@@ -89,7 +90,8 @@ class Commands {
     static parse(samplePath, outPath, generateMarkdown) {
         const parser = new CommonmarkParser();
         const markdownText = fs.readFileSync(samplePath, 'utf8');
-        const concertoObject = parser.parse(markdownText);
+        let concertoObject = parser.parse(markdownText);
+        concertoObject = CommonmarkToAP(concertoObject);
         let result;
         if (generateMarkdown) {
             result = CommonmarkToString(concertoObject);

src/CommonmarkAP.test.js

@@ -0,0 +1,156 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-nocheck
+/* eslint-disable no-undef */
+'use strict';
+
+const fs = require('fs');
+const diff = require('jest-diff');
+const CommonmarkParser = require('./CommonmarkParser');
+const CommonmarkToAP = require('./CommonmarkToAP');
+const CommonmarkToString = require('./CommonmarkToString');
+let parser = null;
+
+expect.extend({
+    toMarkdownRoundtrip(markdownText) {
+        let concertoObject1 = parser.parse(markdownText);
+        concertoObject1 = CommonmarkToAP(concertoObject1);
+        const json1 = parser.getSerializer().toJSON(concertoObject1);
+        const newMarkdown = CommonmarkToString(concertoObject1);
+        let concertoObject2 = parser.parse(newMarkdown);
+        concertoObject2 = CommonmarkToAP(concertoObject2);
+        const json2 = parser.getSerializer().toJSON(concertoObject2);
+        const pass = JSON.stringify(json1) === JSON.stringify(json2);
+
+        const message = pass
+            ? () =>
+                this.utils.matcherHint(`toMarkdownRoundtrip - ${markdownText} -> ${newMarkdown}`, undefined, undefined, undefined) +
+          '\n\n' +
+          `Expected: ${this.utils.printExpected(json1)}\n` +
+          `Received: ${this.utils.printReceived(json2)}`
+            : () => {
+                const diffString = diff(json1, json2, {
+                    expand: true,
+                });
+                return (
+                    this.utils.matcherHint(`toMarkdownRoundtrip - ${JSON.stringify(markdownText)} -> ${JSON.stringify(newMarkdown)}`, undefined, undefined, undefined) +
+            '\n\n' +
+            (diffString && diffString.includes('- Expect')
+                ? `Difference:\n\n${diffString}`
+                : `Expected: ${this.utils.printExpected(json1)}\n` +
+                `Received: ${this.utils.printReceived(json2)}`)
+                );
+            };
+
+        return {actual: markdownText, message, pass};
+    },
+});
+
+// @ts-ignore
+beforeAll(() => {
+    parser = new CommonmarkParser();
+});
+
+/**
+ * Get the name and contents of all markdown test files
+ * @returns {*} an array of name/contents tuples
+ */
+function getMarkdownFiles() {
+    const result = [];
+    const files = fs.readdirSync(__dirname + '/../test/');
+
+    files.forEach(function(file) {
+        if(file.endsWith('.md')) {
+            let contents = fs.readFileSync(__dirname + '/../test/' + file, 'utf8');
+            result.push([file, contents]);
+        }
+    });
+
+    return result;
+}
+
+/**
+ * Get the name and contents of all markdown snippets
+ * used in a commonmark spec file
+ * @returns {*} an array of name/contents tuples
+ */
+function getMarkdownSpecFiles() {
+    const result = [];
+    const specExamples = extractSpecTests(__dirname + '/../test/spec.txt');
+    specExamples.forEach(function(example) {
+        result.push([`${example.section}-${example.number}`, example.markdown]);
+    });
+
+    return result;
+}
+
+/**
+ * Extracts all the test md snippets from a commonmark spec file
+ * @param {string} testfile the file to use
+ * @return {*} the examples
+ */
+function extractSpecTests(testfile) {
+    let data = fs.readFileSync(testfile, 'utf8');
+    let examples = [];
+    let current_section = '';
+    let example_number = 0;
+    let tests = data
+        .replace(/\r\n?/g, '\n') // Normalize newlines for platform independence
+        .replace(/^<!-- END TESTS -->(.|[\n])*/m, '');
+
+    tests.replace(/^`{32} example\n([\s\S]*?)^\.\n([\s\S]*?)^`{32}$|^#{1,6} *(.*)$/gm,
+        function(_, markdownSubmatch, htmlSubmatch, sectionSubmatch){
+            if (sectionSubmatch) {
+                current_section = sectionSubmatch;
+            } else {
+                example_number++;
+                examples.push({markdown: markdownSubmatch,
+                    html: htmlSubmatch,
+                    section: current_section,
+                    number: example_number});
+            }
+        });
+    return examples;
+}
+
+describe('markdown', () => {
+    getMarkdownFiles().forEach( ([file, markdownText]) => {
+        it(`converts ${file} to concerto`, () => {
+            const concertoObject = parser.parse(markdownText);
+            const json = parser.getSerializer().toJSON(concertoObject);
+            expect(json).toMatchSnapshot();
+        });
+
+        it(`roundtrips ${file}`, () => {
+            expect(markdownText).toMarkdownRoundtrip();
+        });
+    });
+});
+
+describe('markdown-spec', () => {
+    getMarkdownSpecFiles().forEach( ([file, markdownText]) => {
+        it(`converts ${file} to concerto`, () => {
+            const concertoObject = parser.parse(markdownText);
+            const json = parser.getSerializer().toJSON(concertoObject);
+            expect(json).toMatchSnapshot();
+        });
+
+        // currently skipped because not all examples roundtrip
+        // needs more investigation!!
+        it.skip(`roundtrips ${file}`, () => {
+            expect(markdownText).toMarkdownRoundtrip();
+        });
+    });
+});

src/CommonmarkParser.js

@@ -16,13 +16,12 @@
 
 const commonmark = require('commonmark');
 const sax = require('sax');
-const NS_PREFIX = 'org.accordproject.commonmark.';
 const ModelManager = require('composer-concerto').ModelManager;
 const Factory = require('composer-concerto').Factory;
 const Serializer = require('composer-concerto').Serializer;
 const Stack = require('./Stack');
 const { DOMParser } = require('xmldom');
-const { commonmarkModel } = require('./Models');
+const { NS_PREFIX, commonmarkModel } = require('./Models');
 
 /**
  * Parses markdown using the commonmark parser into the

src/CommonmarkToAP.js

@@ -0,0 +1,50 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const ModelManager = require('composer-concerto').ModelManager;
+const Factory = require('composer-concerto').Factory;
+const Serializer = require('composer-concerto').Serializer;
+
+const ToAPVisitor = require('./ToAPVisitor');
+const { commonmarkModel } = require('./Models');
+
+/**
+ * Converts a commonmark document to a markdown string
+ * @param {*} concertoObject concerto commonmark object
+ * @returns {*} concertoObject concerto commonmark for AP object
+ */
+function commonMarkToAP(concertoObject) {
+    // Setup for validation
+    const modelManager = new ModelManager();
+    modelManager.addModelFile( commonmarkModel, 'commonmark.cto');
+    const factory = new Factory(modelManager);
+    const serializer = new Serializer(factory, modelManager);
+
+    // Add AP nodes
+    const parameters = {
+        modelManager : modelManager ,
+        factory : factory,
+        serializer : serializer
+    };
+    const visitor = new ToAPVisitor();
+    concertoObject.accept( visitor, parameters );
+
+    // Validate
+    const json = serializer.toJSON(concertoObject);
+    return serializer.fromJSON(json);
+}
+
+module.exports = commonMarkToAP;

src/Models.js

@@ -14,6 +14,7 @@
 
 'use strict';
 
+const NS_PREFIX = 'org.accordproject.commonmark.';
 
 const commonmarkModel = `
 namespace org.accordproject.commonmark
@@ -57,6 +58,11 @@ concept CodeBlock extends Child {
     o TagInfo tag optional
 }
 
+concept Clause extends CodeBlock {
+    o String clauseid
+    o String src
+}
+
 concept Code extends Child {
     o String info optional
 }
@@ -65,6 +71,15 @@ concept HtmlInline extends Child {
     o TagInfo tag optional
 }
 
+concept Variable extends HtmlInline {
+    o String id
+    o String value
+}
+
+concept ComputedVariable extends HtmlInline {
+    o String value
+}
+
 concept HtmlBlock extends Child {
     o TagInfo tag optional
 }
@@ -119,4 +134,4 @@ concept Document extends Root {
 }
 `;
 
-module.exports = { commonmarkModel };
+module.exports = { NS_PREFIX, commonmarkModel };

src/ToAPVisitor.js

@@ -0,0 +1,104 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const { NS_PREFIX } = require('./Models');
+
+/**
+ * Converts a commonmark model instance to a markdown string.
+ *
+ * Note that there are several ways of representing the same markdown AST as text,
+ * so this transformation is not guaranteed to equivalent if you roundtrip
+ * markdown content. The resulting AST *should* be equivalent however.
+ */
+class ToAPVisitor {
+
+    /**
+     * Visits a sub-tree and return the markdown
+     * @param {*} visitor the visitor to use
+     * @param {*} thing the node to visit
+     * @param {*} [parameters] optional parameters
+     */
+    static visitChildren(visitor, thing, parameters) {
+        if(thing.nodes) {
+            thing.nodes.forEach(node => {
+                node.accept(visitor, parameters);
+            });
+        }
+    }
+
+    /**
+     * Visit a node
+     * @param {*} thing the object being visited
+     * @param {*} parameters the parameters
+     */
+    visit(thing, parameters) {
+        switch(thing.getType()) {
+        case 'CodeBlock':
+            if (thing.tag && thing.tag.tagName === 'clause' && thing.tag.attributes.length === 2) {
+                const tag = thing.tag;
+                if (tag.attributes[0].name === 'src' &&
+                    tag.attributes[1].name === 'clauseid') {
+                    thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Clause');
+                    thing.src = tag.attributes[0].name;
+                    thing.clauseid = tag.attributes[1].name;
+                }
+                else if (tag.attributes[1].name === 'src' &&
+                         tag.attributes[0].name === 'clauseid') {
+                    thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Clause');
+                    thing.clauseid = tag.attributes[0].name;
+                    thing.src = tag.attributes[1].name;
+                } else {
+                    //console.log('Found Clause but without \'clauseid\' and \'src\' attributes ');
+                }
+            }
+            break;
+        case 'HtmlInline':
+        //case 'HtmlBlock':
+            if (thing.tag && thing.tag.tagName === 'variable' && thing.tag.attributes.length === 2) {
+                const tag = thing.tag;
+                if (tag.attributes[0].name === 'id' &&
+                    tag.attributes[1].name === 'value') {
+                    thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Variable');
+                    thing.id = tag.attributes[0].name;
+                    thing.value = tag.attributes[1].name;
+                }
+                else if (tag.attributes[1].name === 'id' &&
+                         tag.attributes[0].name === 'value') {
+                    thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Clause');
+                    thing.value = tag.attributes[0].name;
+                    thing.id  = tag.attributes[1].name;
+                } else {
+                    //console.log('Found Variable but without \'id\' and \'value\' attributes ');
+                }
+            }
+            if (thing.tag && thing.tag.tagName === 'computed' && thing.tag.attributes.length === 1) {
+                const tag = thing.tag;
+                if (tag.attributes[0].name === 'value') {
+                    thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'ComputedVariable');
+                    thing.value = tag.attributes[0].name;
+                }
+                else {
+                    //console.log('Found ComputedVariable but without \'value\' attributes ');
+                }
+            }
+            break;
+        default:
+            ToAPVisitor.visitChildren(this, thing, parameters);
+        }
+    }
+}
+
+module.exports = ToAPVisitor;