Breaking: use Hercule for transclusion

Internally Hercule tokenises files into content and link tokens. Only the link portion (excluding the identifying surrounding syntax) is parsed. In the simple case this is simply a file path.In this regard, Aglio syntax permits a strict subset of Hercule's with differing surrounding markers.

Hercule also has built-in circular dependency detection, to prevent the it hanging when provided with bad input.

- Remove: INCLUDE regular expression.
- New: add tokenising regular expression, link match function to extend Hercule's tokeniser with support for existing Aglio syntax.
- Update: compileFile, render to use hercule.transcludeString.
- Beaking: collectPathsSync removed in favour of collectPaths (async).
Hercule is written using streams, which are asynchronous. Without extension of Hercule's sync API (childProcess.spawnSync) which is significantly less performant, collectPathsSync becomes collectPaths (async).
- Docs: updated collectPaths to reflect async.

README.md

@@ -156,12 +156,16 @@ aglio.render(blueprint, options, function (err, html, warnings) {
 ### Reference
 The following methods are available from the `aglio` library:
 
-#### aglio.collectPathsSync (blueprint, includePath)
+#### aglio.collectPaths (blueprint, includePath, callback)
 Get a list of paths that are included in the blueprint. This list can be watched for changes to do things like live reload. The blueprint's own path is not included.
 
 ```javascript
 var blueprint = '# GET /foo\n<-- include(example.json -->\n';
-var watchPaths = aglio.collectPathsSync(blueprint, process.cwd())
+aglio.collectPaths(blueprint, process.cwd(), function (err, watchPaths) {
+  if (err) return console.log(err);
+
+  console.log(watchPaths);
+});
 ```
 
 #### aglio.render (blueprint, options, callback)

example.apib

@@ -46,7 +46,7 @@ Some non-standard Markdown extensions are also supported, such as this informati
 These extensions may change in the future as the [CommonMark specification](http://spec.commonmark.org/) defines a [standard extension syntax](https://github.com/jgm/CommonMark/wiki/Proposed-Extensions).
 :::
 
-<!-- include(example-include.md) -->
+:[Hercule Syntax Include](example-include.md)
 
 # Data Structures
 

package.json

@@ -38,6 +38,7 @@
     "aglio-theme-olio": "^1.6.2",
     "chokidar": "^1.4.1",
     "cli-color": "^1.1.0",
+    "hercule": "^3.0.1",
     "pretty-error": "^1.2.0",
     "protagonist": "^1.2.2",
     "serve-static": "^1.10.0",

src/bin.coffee

@@ -146,15 +146,15 @@ exports.run = (argv=parser.argv, done=->) ->
             socket.on 'request-refresh', ->
                 sendHtml socket
 
-        paths = aglio.collectPathsSync fs.readFileSync(argv.i, 'utf-8'), path.dirname(argv.i)
+        aglio.collectPaths fs.readFileSync(argv.i, 'utf-8'), path.dirname(argv.i), (err, paths) ->
+            watcher = chokidar.watch [argv.i].concat(paths)
+            watcher.on "change", (path) ->
+                console.log "Updated " + path
+                _html = null
+                sendHtml io
 
-        watcher = chokidar.watch [argv.i].concat(paths)
-        watcher.on "change", (path) ->
-            console.log "Updated " + path
-            _html = null
-            sendHtml io
+            done()
 
-        done()
     else
         # Render or Compile API Blueprint, requires input/output files
         if not argv.i or not argv.o

src/main.coffee

@@ -1,8 +1,11 @@
 fs = require 'fs'
 path = require 'path'
 protagonist = require 'protagonist'
+hercule = require 'hercule'
+
+linkRegExp = new RegExp(/(^[\t ]*)?(?:(\:\[.*?\]\((.*?)\))|(<!-- include\((.*?)\) -->))/gm)
+linkMatch = (match) -> match[3] || match[5]
 
-INCLUDE = /( *)<!-- include\((.*)\) -->/gmi
 ROOT = path.dirname __dirname
 
 # Legacy template names
@@ -20,34 +23,13 @@ errMsg = (message, err) ->
     err.message = "#{message}: #{err.message}"
     return err
 
-# Replace the include directive with the contents of the included
-# file in the input.
-includeReplace = (includePath, match, spaces, filename) ->
-    fullPath = path.join includePath, filename
-    lines = fs.readFileSync(fullPath, 'utf-8').replace(/\r\n?/g, '\n').split('\n')
-    content = spaces + lines.join "\n#{spaces}"
-
-    # The content can itself include other files, so check those
-    # as well! Beware of circular includes!
-    includeDirective path.dirname(fullPath), content
-
-# Handle the include directive, which inserts the contents of one
-# file into another. We find the directive using a regular expression
-# and replace it using the method above.
-includeDirective = (includePath, input) ->
-    input.replace INCLUDE, includeReplace.bind(this, includePath)
-
 # Get a list of all paths from included files. This *excludes* the
 # input path itself.
-exports.collectPathsSync = (input, includePath) ->
+exports.collectPaths = (input, includePath, done) ->
     paths = []
-    input.replace INCLUDE, (match, spaces, filename) ->
-        fullPath = path.join(includePath, filename)
-        paths.push fullPath
-
-        content = fs.readFileSync fullPath, 'utf-8'
-        paths = paths.concat exports.collectPathsSync(content, path.dirname(fullPath))
-    paths
+    hercule.transcludeString input, {relativePath: includePath, linkRegExp, linkMatch}, paths, (err, output) ->
+        if err then return done(err)
+        return done(null, paths)
 
 # Get the theme module for a given theme name
 exports.getTheme = (name) ->
@@ -80,45 +62,44 @@ exports.render = (input, options, done) ->
         options.theme = 'olio'
 
     # Handle custom directive(s)
-    input = includeDirective options.includePath, input
-
-    # Protagonist does not support \r ot \t in the input, so
-    # try to intelligently massage the input so that it works.
-    # This is required to process files created on Windows.
-    filteredInput = if not options.filterInput then input else
-        input
-            .replace(/\r\n?/g, '\n')
-            .replace(/\t/g, '    ')
-
-    benchmark.start 'parse'
-    protagonist.parse filteredInput, type: 'ast', (err, res) ->
-        benchmark.end 'parse'
-        if err
-            err.input = input
-            return done(errMsg 'Error parsing input', err)
-
-        try
-            theme = exports.getTheme options.theme
-        catch err
-            return done(errMsg 'Error getting theme', err)
-
-        # Setup default options if needed
-        for option in theme.getConfig().options or []
-            # Convert `foo-bar` into `themeFooBar`
-            words = (f[0].toUpperCase() + f.slice(1) for f in option.name.split('-'))
-            name = "theme#{words.join('')}"
-            options[name] ?= option.default
-
-        benchmark.start 'render-total'
-        theme.render res.ast, options, (err, html) ->
-            benchmark.end 'render-total'
-            if err then return done(err)
-
-            # Add filtered input to warnings since we have no
-            # error to return
-            res.warnings.input = filteredInput
-
-            done null, html, res.warnings
+    hercule.transcludeString input, {relativePath: options.includePath, linkRegExp, linkMatch}, (err, input) ->
+        # Protagonist does not support \r ot \t in the input, so
+        # try to intelligently massage the input so that it works.
+        # This is required to process files created on Windows.
+        filteredInput = if not options.filterInput then input else
+            input
+                .replace(/\r\n?/g, '\n')
+                .replace(/\t/g, '    ')
+
+        benchmark.start 'parse'
+        protagonist.parse filteredInput, type: 'ast', (err, res) ->
+            benchmark.end 'parse'
+            if err
+                err.input = input
+                return done(errMsg 'Error parsing input', err)
+
+            try
+                theme = exports.getTheme options.theme
+            catch err
+                return done(errMsg 'Error getting theme', err)
+
+            # Setup default options if needed
+            for option in theme.getConfig().options or []
+                # Convert `foo-bar` into `themeFooBar`
+                words = (f[0].toUpperCase() + f.slice(1) for f in option.name.split('-'))
+                name = "theme#{words.join('')}"
+                options[name] ?= option.default
+
+            benchmark.start 'render-total'
+            theme.render res.ast, options, (err, html) ->
+                benchmark.end 'render-total'
+                if err then return done(err)
+
+                # Add filtered input to warnings since we have no
+                # error to return
+                res.warnings.input = filteredInput
+
+                done null, html, res.warnings
 
 # Render from/to files
 exports.renderFile = (inputFile, outputFile, options, done) ->
@@ -148,14 +129,14 @@ exports.renderFile = (inputFile, outputFile, options, done) ->
 # Compile markdown from/to files
 exports.compileFile = (inputFile, outputFile, done) ->
     compile = (input) ->
-        compiled = includeDirective path.dirname(inputFile), input
-
-        if outputFile isnt '-'
-            fs.writeFile outputFile, compiled, (err) ->
-                done err
-        else
-            console.log compiled
-            done null
+        hercule.transcludeString input, { linkRegExp, linkMatch }, (err, compiled) ->
+            if err then return done(errMsg 'Error compiling output', err)
+            if outputFile isnt '-'
+                fs.writeFile outputFile, compiled, (err) ->
+                    done err
+            else
+                console.log compiled
+                done null
 
     if inputFile isnt '-'
         fs.readFile inputFile, encoding: 'utf-8', (err, input) ->

test/basic.coffee

@@ -17,9 +17,15 @@ describe 'API Blueprint Renderer', ->
 
         assert.ok theme
 
-    it 'Should get a list of included files', ->
-        sinon.stub fs, 'readFileSync', -> 'I am a test file'
+    it 'Should get a list of included files', (done) ->
+        aglio.collectPaths blueprint, '.', (err, paths) ->
+            assert.equal err, null
+            assert.equal paths.length, 2
+            assert 'example-include.md' in paths
+            assert 'example-schema.json' in paths
+            done()
 
+    it 'Should return error on bad links', (done) ->
         input = '''
             # Title
             <!-- include(test1.apib) -->
@@ -28,13 +34,9 @@ describe 'API Blueprint Renderer', ->
             More content...
         '''
 
-        paths = aglio.collectPathsSync input, '.'
-
-        fs.readFileSync.restore()
-
-        assert.equal paths.length, 2
-        assert 'test1.apib' in paths
-        assert 'test2.apib' in paths
+        aglio.collectPaths input, '.', (err, paths) ->
+            assert.equal err.msg, 'Could not read file'
+            done()
 
     it 'Should render blank string', (done) ->
         aglio.render '', template: 'default', locals: {foo: 1}, (err, html) ->
@@ -130,6 +132,15 @@ describe 'API Blueprint Renderer', ->
 
             done()
 
+    it 'Should handle compile errors', (done) ->
+        sinon.stub process.stdin, 'read', -> '# Hello\n<!-- include(i-dont-exist.md) -->\n'
+
+        setTimeout -> process.stdin.emit 'readable', 1
+
+        aglio.compileFile '-', 'example-compiled.apib', (err) ->
+            assert.equal err.msg, 'Could not read file'
+            done()
+
     it 'Should compile to stdout', (done) ->
         sinon.stub console, 'log'