From 06808be4de29f6db523128e96f8c7fb35c5412af Mon Sep 17 00:00:00 2001 From: James Ramsay Date: Fri, 11 Mar 2016 23:25:10 +1100 Subject: [PATCH] Breaking: use Hercule for transclusion Internally Hercule tokenises files into content and link tokens. Only the link portion (excluding the identifying surrounding syntax) is parsed. In the simple case this is simply a file path.In this regard, Aglio syntax permits a strict subset of Hercule's with differing surrounding markers. Hercule also has built-in circular dependency detection, to prevent the it hanging when provided with bad input. - Remove: INCLUDE regular expression. - New: add tokenising regular expression, link match function to extend Hercule's tokeniser with support for existing Aglio syntax. - Update: compileFile, render to use hercule.transcludeString. - Beaking: collectPathsSync removed in favour of collectPaths (async). Hercule is written using streams, which are asynchronous. Without extension of Hercule's sync API (childProcess.spawnSync) which is significantly less performant, collectPathsSync becomes collectPaths (async). - Docs: updated collectPaths to reflect async. --- README.md | 8 ++- example.apib | 2 +- package.json | 1 + src/bin.coffee | 14 ++--- src/main.coffee | 127 ++++++++++++++++++++-------------------------- test/basic.coffee | 29 +++++++---- 6 files changed, 89 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index 6626ce06..83b05b99 100644 --- a/README.md +++ b/README.md @@ -156,12 +156,16 @@ aglio.render(blueprint, options, function (err, html, warnings) { ### Reference The following methods are available from the `aglio` library: -#### aglio.collectPathsSync (blueprint, includePath) +#### aglio.collectPaths (blueprint, includePath, callback) Get a list of paths that are included in the blueprint. This list can be watched for changes to do things like live reload. The blueprint's own path is not included. ```javascript var blueprint = '# GET /foo\n<-- include(example.json -->\n'; -var watchPaths = aglio.collectPathsSync(blueprint, process.cwd()) +aglio.collectPaths(blueprint, process.cwd(), function (err, watchPaths) { + if (err) return console.log(err); + + console.log(watchPaths); +}); ``` #### aglio.render (blueprint, options, callback) diff --git a/example.apib b/example.apib index 44ff9692..6f6a0fba 100644 --- a/example.apib +++ b/example.apib @@ -46,7 +46,7 @@ Some non-standard Markdown extensions are also supported, such as this informati These extensions may change in the future as the [CommonMark specification](http://spec.commonmark.org/) defines a [standard extension syntax](https://github.com/jgm/CommonMark/wiki/Proposed-Extensions). ::: - +:[Hercule Syntax Include](example-include.md) # Data Structures diff --git a/package.json b/package.json index fb840352..70d1122d 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "aglio-theme-olio": "^1.6.2", "chokidar": "^1.4.1", "cli-color": "^1.1.0", + "hercule": "^3.0.1", "pretty-error": "^1.2.0", "protagonist": "^1.2.2", "serve-static": "^1.10.0", diff --git a/src/bin.coffee b/src/bin.coffee index 977f6111..bab0662c 100644 --- a/src/bin.coffee +++ b/src/bin.coffee @@ -146,15 +146,15 @@ exports.run = (argv=parser.argv, done=->) -> socket.on 'request-refresh', -> sendHtml socket - paths = aglio.collectPathsSync fs.readFileSync(argv.i, 'utf-8'), path.dirname(argv.i) + aglio.collectPaths fs.readFileSync(argv.i, 'utf-8'), path.dirname(argv.i), (err, paths) -> + watcher = chokidar.watch [argv.i].concat(paths) + watcher.on "change", (path) -> + console.log "Updated " + path + _html = null + sendHtml io - watcher = chokidar.watch [argv.i].concat(paths) - watcher.on "change", (path) -> - console.log "Updated " + path - _html = null - sendHtml io + done() - done() else # Render or Compile API Blueprint, requires input/output files if not argv.i or not argv.o diff --git a/src/main.coffee b/src/main.coffee index da860cca..4c735952 100644 --- a/src/main.coffee +++ b/src/main.coffee @@ -1,8 +1,11 @@ fs = require 'fs' path = require 'path' protagonist = require 'protagonist' +hercule = require 'hercule' + +linkRegExp = new RegExp(/(^[\t ]*)?(?:(\:\[.*?\]\((.*?)\))|())/gm) +linkMatch = (match) -> match[3] || match[5] -INCLUDE = /( *)/gmi ROOT = path.dirname __dirname # Legacy template names @@ -20,34 +23,13 @@ errMsg = (message, err) -> err.message = "#{message}: #{err.message}" return err -# Replace the include directive with the contents of the included -# file in the input. -includeReplace = (includePath, match, spaces, filename) -> - fullPath = path.join includePath, filename - lines = fs.readFileSync(fullPath, 'utf-8').replace(/\r\n?/g, '\n').split('\n') - content = spaces + lines.join "\n#{spaces}" - - # The content can itself include other files, so check those - # as well! Beware of circular includes! - includeDirective path.dirname(fullPath), content - -# Handle the include directive, which inserts the contents of one -# file into another. We find the directive using a regular expression -# and replace it using the method above. -includeDirective = (includePath, input) -> - input.replace INCLUDE, includeReplace.bind(this, includePath) - # Get a list of all paths from included files. This *excludes* the # input path itself. -exports.collectPathsSync = (input, includePath) -> +exports.collectPaths = (input, includePath, done) -> paths = [] - input.replace INCLUDE, (match, spaces, filename) -> - fullPath = path.join(includePath, filename) - paths.push fullPath - - content = fs.readFileSync fullPath, 'utf-8' - paths = paths.concat exports.collectPathsSync(content, path.dirname(fullPath)) - paths + hercule.transcludeString input, {relativePath: includePath, linkRegExp, linkMatch}, paths, (err, output) -> + if err then return done(err) + return done(null, paths) # Get the theme module for a given theme name exports.getTheme = (name) -> @@ -80,45 +62,44 @@ exports.render = (input, options, done) -> options.theme = 'olio' # Handle custom directive(s) - input = includeDirective options.includePath, input - - # Protagonist does not support \r ot \t in the input, so - # try to intelligently massage the input so that it works. - # This is required to process files created on Windows. - filteredInput = if not options.filterInput then input else - input - .replace(/\r\n?/g, '\n') - .replace(/\t/g, ' ') - - benchmark.start 'parse' - protagonist.parse filteredInput, type: 'ast', (err, res) -> - benchmark.end 'parse' - if err - err.input = input - return done(errMsg 'Error parsing input', err) - - try - theme = exports.getTheme options.theme - catch err - return done(errMsg 'Error getting theme', err) - - # Setup default options if needed - for option in theme.getConfig().options or [] - # Convert `foo-bar` into `themeFooBar` - words = (f[0].toUpperCase() + f.slice(1) for f in option.name.split('-')) - name = "theme#{words.join('')}" - options[name] ?= option.default - - benchmark.start 'render-total' - theme.render res.ast, options, (err, html) -> - benchmark.end 'render-total' - if err then return done(err) - - # Add filtered input to warnings since we have no - # error to return - res.warnings.input = filteredInput - - done null, html, res.warnings + hercule.transcludeString input, {relativePath: options.includePath, linkRegExp, linkMatch}, (err, input) -> + # Protagonist does not support \r ot \t in the input, so + # try to intelligently massage the input so that it works. + # This is required to process files created on Windows. + filteredInput = if not options.filterInput then input else + input + .replace(/\r\n?/g, '\n') + .replace(/\t/g, ' ') + + benchmark.start 'parse' + protagonist.parse filteredInput, type: 'ast', (err, res) -> + benchmark.end 'parse' + if err + err.input = input + return done(errMsg 'Error parsing input', err) + + try + theme = exports.getTheme options.theme + catch err + return done(errMsg 'Error getting theme', err) + + # Setup default options if needed + for option in theme.getConfig().options or [] + # Convert `foo-bar` into `themeFooBar` + words = (f[0].toUpperCase() + f.slice(1) for f in option.name.split('-')) + name = "theme#{words.join('')}" + options[name] ?= option.default + + benchmark.start 'render-total' + theme.render res.ast, options, (err, html) -> + benchmark.end 'render-total' + if err then return done(err) + + # Add filtered input to warnings since we have no + # error to return + res.warnings.input = filteredInput + + done null, html, res.warnings # Render from/to files exports.renderFile = (inputFile, outputFile, options, done) -> @@ -148,14 +129,14 @@ exports.renderFile = (inputFile, outputFile, options, done) -> # Compile markdown from/to files exports.compileFile = (inputFile, outputFile, done) -> compile = (input) -> - compiled = includeDirective path.dirname(inputFile), input - - if outputFile isnt '-' - fs.writeFile outputFile, compiled, (err) -> - done err - else - console.log compiled - done null + hercule.transcludeString input, { linkRegExp, linkMatch }, (err, compiled) -> + if err then return done(errMsg 'Error compiling output', err) + if outputFile isnt '-' + fs.writeFile outputFile, compiled, (err) -> + done err + else + console.log compiled + done null if inputFile isnt '-' fs.readFile inputFile, encoding: 'utf-8', (err, input) -> diff --git a/test/basic.coffee b/test/basic.coffee index 5a1a407b..f51e90eb 100644 --- a/test/basic.coffee +++ b/test/basic.coffee @@ -17,9 +17,15 @@ describe 'API Blueprint Renderer', -> assert.ok theme - it 'Should get a list of included files', -> - sinon.stub fs, 'readFileSync', -> 'I am a test file' + it 'Should get a list of included files', (done) -> + aglio.collectPaths blueprint, '.', (err, paths) -> + assert.equal err, null + assert.equal paths.length, 2 + assert 'example-include.md' in paths + assert 'example-schema.json' in paths + done() + it 'Should return error on bad links', (done) -> input = ''' # Title @@ -28,13 +34,9 @@ describe 'API Blueprint Renderer', -> More content... ''' - paths = aglio.collectPathsSync input, '.' - - fs.readFileSync.restore() - - assert.equal paths.length, 2 - assert 'test1.apib' in paths - assert 'test2.apib' in paths + aglio.collectPaths input, '.', (err, paths) -> + assert.equal err.msg, 'Could not read file' + done() it 'Should render blank string', (done) -> aglio.render '', template: 'default', locals: {foo: 1}, (err, html) -> @@ -130,6 +132,15 @@ describe 'API Blueprint Renderer', -> done() + it 'Should handle compile errors', (done) -> + sinon.stub process.stdin, 'read', -> '# Hello\n\n' + + setTimeout -> process.stdin.emit 'readable', 1 + + aglio.compileFile '-', 'example-compiled.apib', (err) -> + assert.equal err.msg, 'Could not read file' + done() + it 'Should compile to stdout', (done) -> sinon.stub console, 'log'