diff --git a/src/Commands.js b/src/Commands.js index a6791401..4c296407 100644 --- a/src/Commands.js +++ b/src/Commands.js @@ -18,6 +18,7 @@ const fs = require('fs'); const Logger = require('./Logger'); const CommonmarkParser = require('./CommonmarkParser'); const CommonmarkToString = require('./CommonmarkToString'); +const CommonmarkToAP = require('./CommonmarkToAP'); /** * Utility class that implements the commands exposed by the CLI. @@ -89,7 +90,8 @@ class Commands { static parse(samplePath, outPath, generateMarkdown) { const parser = new CommonmarkParser(); const markdownText = fs.readFileSync(samplePath, 'utf8'); - const concertoObject = parser.parse(markdownText); + let concertoObject = parser.parse(markdownText); + concertoObject = CommonmarkToAP(concertoObject); let result; if (generateMarkdown) { result = CommonmarkToString(concertoObject); diff --git a/src/CommonmarkAP.test.js b/src/CommonmarkAP.test.js new file mode 100644 index 00000000..9ddb4259 --- /dev/null +++ b/src/CommonmarkAP.test.js @@ -0,0 +1,156 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-nocheck +/* eslint-disable no-undef */ +'use strict'; + +const fs = require('fs'); +const diff = require('jest-diff'); +const CommonmarkParser = require('./CommonmarkParser'); +const CommonmarkToAP = require('./CommonmarkToAP'); +const CommonmarkToString = require('./CommonmarkToString'); +let parser = null; + +expect.extend({ + toMarkdownRoundtrip(markdownText) { + let concertoObject1 = parser.parse(markdownText); + concertoObject1 = CommonmarkToAP(concertoObject1); + const json1 = parser.getSerializer().toJSON(concertoObject1); + const newMarkdown = CommonmarkToString(concertoObject1); + let concertoObject2 = parser.parse(newMarkdown); + concertoObject2 = CommonmarkToAP(concertoObject2); + const json2 = parser.getSerializer().toJSON(concertoObject2); + const pass = JSON.stringify(json1) === JSON.stringify(json2); + + const message = pass + ? () => + this.utils.matcherHint(`toMarkdownRoundtrip - ${markdownText} -> ${newMarkdown}`, undefined, undefined, undefined) + + '\n\n' + + `Expected: ${this.utils.printExpected(json1)}\n` + + `Received: ${this.utils.printReceived(json2)}` + : () => { + const diffString = diff(json1, json2, { + expand: true, + }); + return ( + this.utils.matcherHint(`toMarkdownRoundtrip - ${JSON.stringify(markdownText)} -> ${JSON.stringify(newMarkdown)}`, undefined, undefined, undefined) + + '\n\n' + + (diffString && diffString.includes('- Expect') + ? `Difference:\n\n${diffString}` + : `Expected: ${this.utils.printExpected(json1)}\n` + + `Received: ${this.utils.printReceived(json2)}`) + ); + }; + + return {actual: markdownText, message, pass}; + }, +}); + +// @ts-ignore +beforeAll(() => { + parser = new CommonmarkParser(); +}); + +/** + * Get the name and contents of all markdown test files + * @returns {*} an array of name/contents tuples + */ +function getMarkdownFiles() { + const result = []; + const files = fs.readdirSync(__dirname + '/../test/'); + + files.forEach(function(file) { + if(file.endsWith('.md')) { + let contents = fs.readFileSync(__dirname + '/../test/' + file, 'utf8'); + result.push([file, contents]); + } + }); + + return result; +} + +/** + * Get the name and contents of all markdown snippets + * used in a commonmark spec file + * @returns {*} an array of name/contents tuples + */ +function getMarkdownSpecFiles() { + const result = []; + const specExamples = extractSpecTests(__dirname + '/../test/spec.txt'); + specExamples.forEach(function(example) { + result.push([`${example.section}-${example.number}`, example.markdown]); + }); + + return result; +} + +/** + * Extracts all the test md snippets from a commonmark spec file + * @param {string} testfile the file to use + * @return {*} the examples + */ +function extractSpecTests(testfile) { + let data = fs.readFileSync(testfile, 'utf8'); + let examples = []; + let current_section = ''; + let example_number = 0; + let tests = data + .replace(/\r\n?/g, '\n') // Normalize newlines for platform independence + .replace(/^(.|[\n])*/m, ''); + + tests.replace(/^`{32} example\n([\s\S]*?)^\.\n([\s\S]*?)^`{32}$|^#{1,6} *(.*)$/gm, + function(_, markdownSubmatch, htmlSubmatch, sectionSubmatch){ + if (sectionSubmatch) { + current_section = sectionSubmatch; + } else { + example_number++; + examples.push({markdown: markdownSubmatch, + html: htmlSubmatch, + section: current_section, + number: example_number}); + } + }); + return examples; +} + +describe('markdown', () => { + getMarkdownFiles().forEach( ([file, markdownText]) => { + it(`converts ${file} to concerto`, () => { + const concertoObject = parser.parse(markdownText); + const json = parser.getSerializer().toJSON(concertoObject); + expect(json).toMatchSnapshot(); + }); + + it(`roundtrips ${file}`, () => { + expect(markdownText).toMarkdownRoundtrip(); + }); + }); +}); + +describe('markdown-spec', () => { + getMarkdownSpecFiles().forEach( ([file, markdownText]) => { + it(`converts ${file} to concerto`, () => { + const concertoObject = parser.parse(markdownText); + const json = parser.getSerializer().toJSON(concertoObject); + expect(json).toMatchSnapshot(); + }); + + // currently skipped because not all examples roundtrip + // needs more investigation!! + it.skip(`roundtrips ${file}`, () => { + expect(markdownText).toMarkdownRoundtrip(); + }); + }); +}); diff --git a/src/CommonmarkParser.js b/src/CommonmarkParser.js index ef7d2b87..6ca4090e 100644 --- a/src/CommonmarkParser.js +++ b/src/CommonmarkParser.js @@ -16,13 +16,12 @@ const commonmark = require('commonmark'); const sax = require('sax'); -const NS_PREFIX = 'org.accordproject.commonmark.'; const ModelManager = require('composer-concerto').ModelManager; const Factory = require('composer-concerto').Factory; const Serializer = require('composer-concerto').Serializer; const Stack = require('./Stack'); const { DOMParser } = require('xmldom'); -const { commonmarkModel } = require('./Models'); +const { NS_PREFIX, commonmarkModel } = require('./Models'); /** * Parses markdown using the commonmark parser into the diff --git a/src/CommonmarkToAP.js b/src/CommonmarkToAP.js new file mode 100644 index 00000000..87430a4d --- /dev/null +++ b/src/CommonmarkToAP.js @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const ModelManager = require('composer-concerto').ModelManager; +const Factory = require('composer-concerto').Factory; +const Serializer = require('composer-concerto').Serializer; + +const ToAPVisitor = require('./ToAPVisitor'); +const { commonmarkModel } = require('./Models'); + +/** + * Converts a commonmark document to a markdown string + * @param {*} concertoObject concerto commonmark object + * @returns {*} concertoObject concerto commonmark for AP object + */ +function commonMarkToAP(concertoObject) { + // Setup for validation + const modelManager = new ModelManager(); + modelManager.addModelFile( commonmarkModel, 'commonmark.cto'); + const factory = new Factory(modelManager); + const serializer = new Serializer(factory, modelManager); + + // Add AP nodes + const parameters = { + modelManager : modelManager , + factory : factory, + serializer : serializer + }; + const visitor = new ToAPVisitor(); + concertoObject.accept( visitor, parameters ); + + // Validate + const json = serializer.toJSON(concertoObject); + return serializer.fromJSON(json); +} + +module.exports = commonMarkToAP; \ No newline at end of file diff --git a/src/Models.js b/src/Models.js index 041e6af5..267251f4 100644 --- a/src/Models.js +++ b/src/Models.js @@ -14,6 +14,7 @@ 'use strict'; +const NS_PREFIX = 'org.accordproject.commonmark.'; const commonmarkModel = ` namespace org.accordproject.commonmark @@ -57,6 +58,11 @@ concept CodeBlock extends Child { o TagInfo tag optional } +concept Clause extends CodeBlock { + o String clauseid + o String src +} + concept Code extends Child { o String info optional } @@ -65,6 +71,15 @@ concept HtmlInline extends Child { o TagInfo tag optional } +concept Variable extends HtmlInline { + o String id + o String value +} + +concept ComputedVariable extends HtmlInline { + o String value +} + concept HtmlBlock extends Child { o TagInfo tag optional } @@ -119,4 +134,4 @@ concept Document extends Root { } `; -module.exports = { commonmarkModel }; +module.exports = { NS_PREFIX, commonmarkModel }; diff --git a/src/ToAPVisitor.js b/src/ToAPVisitor.js new file mode 100644 index 00000000..6648b8a0 --- /dev/null +++ b/src/ToAPVisitor.js @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const { NS_PREFIX } = require('./Models'); + +/** + * Converts a commonmark model instance to a markdown string. + * + * Note that there are several ways of representing the same markdown AST as text, + * so this transformation is not guaranteed to equivalent if you roundtrip + * markdown content. The resulting AST *should* be equivalent however. + */ +class ToAPVisitor { + + /** + * Visits a sub-tree and return the markdown + * @param {*} visitor the visitor to use + * @param {*} thing the node to visit + * @param {*} [parameters] optional parameters + */ + static visitChildren(visitor, thing, parameters) { + if(thing.nodes) { + thing.nodes.forEach(node => { + node.accept(visitor, parameters); + }); + } + } + + /** + * Visit a node + * @param {*} thing the object being visited + * @param {*} parameters the parameters + */ + visit(thing, parameters) { + switch(thing.getType()) { + case 'CodeBlock': + if (thing.tag && thing.tag.tagName === 'clause' && thing.tag.attributes.length === 2) { + const tag = thing.tag; + if (tag.attributes[0].name === 'src' && + tag.attributes[1].name === 'clauseid') { + thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Clause'); + thing.src = tag.attributes[0].name; + thing.clauseid = tag.attributes[1].name; + } + else if (tag.attributes[1].name === 'src' && + tag.attributes[0].name === 'clauseid') { + thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Clause'); + thing.clauseid = tag.attributes[0].name; + thing.src = tag.attributes[1].name; + } else { + //console.log('Found Clause but without \'clauseid\' and \'src\' attributes '); + } + } + break; + case 'HtmlInline': + //case 'HtmlBlock': + if (thing.tag && thing.tag.tagName === 'variable' && thing.tag.attributes.length === 2) { + const tag = thing.tag; + if (tag.attributes[0].name === 'id' && + tag.attributes[1].name === 'value') { + thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Variable'); + thing.id = tag.attributes[0].name; + thing.value = tag.attributes[1].name; + } + else if (tag.attributes[1].name === 'id' && + tag.attributes[0].name === 'value') { + thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'Clause'); + thing.value = tag.attributes[0].name; + thing.id = tag.attributes[1].name; + } else { + //console.log('Found Variable but without \'id\' and \'value\' attributes '); + } + } + if (thing.tag && thing.tag.tagName === 'computed' && thing.tag.attributes.length === 1) { + const tag = thing.tag; + if (tag.attributes[0].name === 'value') { + thing.$classDeclaration = parameters.modelManager.getType(NS_PREFIX + 'ComputedVariable'); + thing.value = tag.attributes[0].name; + } + else { + //console.log('Found ComputedVariable but without \'value\' attributes '); + } + } + break; + default: + ToAPVisitor.visitChildren(this, thing, parameters); + } + } +} + +module.exports = ToAPVisitor; \ No newline at end of file diff --git a/src/ToStringVisitor.js b/src/ToStringVisitor.js index 7a8e5fad..b9f4c6b7 100644 --- a/src/ToStringVisitor.js +++ b/src/ToStringVisitor.js @@ -123,12 +123,16 @@ class ToStringVisitor { switch(thing.getType()) { case 'CodeBlock': + case 'Clause': + ToStringVisitor.newParagraph(parameters); parameters.result += `\`\`\` ${thing.info ? thing.info : ''}\n${thing.text}\`\`\`\n\n`; break; case 'Code': parameters.result += `\`${thing.text}\``; break; case 'HtmlInline': + case 'Variable': + case 'ComputedVariable': parameters.result += thing.text; break; case 'Emph': diff --git a/src/__snapshots__/CommonmarkAP.test.js.snap b/src/__snapshots__/CommonmarkAP.test.js.snap new file mode 100644 index 00000000..7f56f082 --- /dev/null +++ b/src/__snapshots__/CommonmarkAP.test.js.snap @@ -0,0 +1,22224 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`markdown converts blockquote.md to concerto 1`] = ` +Object { + "$class": "org.accordproject.commonmark.Document", + "nodes": Array [ + Object { + "$class": "org.accordproject.commonmark.Paragraph", + "nodes": Array [ + Object { + "$class": "org.accordproject.commonmark.Text", + "text": "This is", + }, + ], + }, + Object { + "$class": "org.accordproject.commonmark.BlockQuote", + "nodes": Array [ + Object { + "$class": "org.accordproject.commonmark.Paragraph", + "nodes": Array [ + Object { + "$class": "org.accordproject.commonmark.Text", + "text": "A quote.", + }, + ], + }, + ], + }, + ], + "xmlns": "http://commonmark.org/xml/1.0", +} +`; + +exports[`markdown converts codeblock.md to concerto 1`] = ` +Object { + "$class": "org.accordproject.commonmark.Document", + "nodes": Array [ + Object { + "$class": "org.accordproject.commonmark.CodeBlock", + "text": "this is a multiline +code +block. +", + }, + ], + "xmlns": "http://commonmark.org/xml/1.0", +} +`; + +exports[`markdown converts codeblock-info.md to concerto 1`] = ` +Object { + "$class": "org.accordproject.commonmark.Document", + "nodes": Array [ + Object { + "$class": "org.accordproject.commonmark.CodeBlock", + "info": "