[Filter] include-md detects and avoids potential endless recursion (#128

)

filters/include-mdfiles.lua

@@ -13,11 +13,16 @@ for each link to local Markdown file in start document:
                     - read link.target (file), process content and append resulting blocks to block list
                     - add new current block (empty Para) for remaining inlines of current block
             - return block list to replace current Para
+
+
+Caveats:
+The same file cannot be included more than once to avoid potential endless recursion.
 ]]--
 
 
 -- vars
-local ROOT = "."    -- absolute path to working directory when starting
+local ROOT = "."        -- absolute path to working directory when starting
+local frontier = {}     -- set of collected links to avoid processing the same file/link several times
 
 
 -- helper
@@ -51,60 +56,75 @@ local function _join_path (include_path, file)
 end
 
 
--- open file, read content, parse recursivly and return list of blocks
+-- if not already processed, apply the filters to the blocks and return the result as a list of blocks
+local function _filter_blocks (blocks, include_path, target)
+    if not frontier[target] then
+        -- remember this file (path w/o '../' - for this reason, this cannot be checked earlier/elsewhere)
+        frontier[target] = true
+
+        -- process this file
+        return blocks:walk({
+            Image = function (img)
+                if _is_local_path(img.src) then
+                    -- prepend current include path to image source
+                    img.src = _join_path(include_path, img.src)
+                    return img
+                end
+            end,
+            Para = _include_md
+        })
+    else
+        io.stderr:write("\t (_handle_file) WARNING: file has been included before '" .. target .. "' ... skipping ... \n")
+        return pandoc.List:new()
+    end
+end
+
+
+-- open file, read content, parse recursively and return list of blocks
 local function _handle_file (lnk)
     local fh = io.open(lnk, "r")
     if not fh then
-        io.stderr:write("\t (_handle_file) cannot open file '" .. lnk .. "' ... skipping ... \n")
+        io.stderr:write("\t (_handle_file) WARNING: cannot open file '" .. lnk .. "' ... skipping ... \n")
         return pandoc.List:new()
     else
-        local content = fh:read "*all"
+        local blocks = pandoc.read(fh:read "*all", "markdown", PANDOC_READER_OPTIONS).blocks
         fh:close()
 
         return pandoc.system.with_working_directory(
             pandoc.path.directory(lnk),    -- may still contain '../'
             function ()
-                local wrkdir = pandoc.system.get_working_directory()    -- same as 'pandoc.path.directory(lnk)' but w/o '../' since Pandoc cd'ed here
-                return process_doc(
-                    pandoc.read(content, "markdown", PANDOC_READER_OPTIONS).blocks,
-                    pandoc.path.make_relative(wrkdir, ROOT))
+                -- same as 'pandoc.path.directory(lnk)' but w/o '../' since Pandoc cd'ed here
+                local include_path = pandoc.path.make_relative(pandoc.system.get_working_directory(), ROOT)
+                local target = _join_path(include_path, pandoc.path.filename(lnk))
+                return _filter_blocks (blocks, include_path, target)
             end)
     end
 end
 
 
--- process Pandoc document (list of blocks): fix image sources (prepend include path), recursive include of links
-function process_doc (blocks, include_path)
-    return blocks:walk({
-        Image = function (img)
-            if _is_local_path(img.src) then
-                -- prepend current include path to image source
-                img.src = _join_path(include_path, img.src)
-                return img
-            end
-        end,
-        Para = function (bl)
-            local block_list = pandoc.List:new()
-            local current_block = pandoc.Para({})
-            block_list:insert(current_block)
+-- to be used as a filter function for pandoc.Para: check each inline in a pandoc.Para block and
+-- split this block into a list of blocks (before and after the links) and splice a list of blocks
+-- for each link to a local Markdown file
+function _include_md (bl)
+    local block_list = pandoc.List:new()
+    local current_block = pandoc.Para({})
+    block_list:insert(current_block)
 
-            for _,i in ipairs(bl.content) do
-                if _is_local_markdown_file_link(i) then
-                    -- process link target
-                    block_list:extend(_handle_file(i.target))
-
-                    -- "close" current block and open new one for any remaining inlines in current block 'bl'
-                    current_block = pandoc.Para({})
-                    block_list:insert(current_block)
-                else
-                    -- copy inline into block content
-                    current_block.content:insert(i)
-                end
-            end
+    for _,i in ipairs(bl.content) do
+        if _is_local_markdown_file_link(i) then
+            -- process link target
+            block_list:extend(_handle_file(i.target))
 
-            return block_list
+            -- "close" current block and open new one for any remaining inlines in current block 'bl'
+            current_block = pandoc.Para({})
+            block_list:insert(current_block)
+        else
+            -- copy inline into block content
+            current_block.content:insert(i)
         end
-    })
+    end
+
+    return block_list
 end
 
 
@@ -113,6 +133,6 @@ function Pandoc (doc)
     -- remember our project root
     ROOT = pandoc.system.get_working_directory()
 
-    -- process all images and links
-    return pandoc.Pandoc(process_doc(doc.blocks, "."), doc.meta)
+    -- process all images and links (recursively)
+    return pandoc.Pandoc(doc.blocks:walk({Para = _include_md}), doc.meta)
 end

filters/test/Makefile

@@ -1,9 +1,10 @@
 DIFF ?= diff --strip-trailing-cr -u
 PANDOC ?= pandoc
 
-FILES_TRANSFORM = readme.md
-FILES_MAKEDEPS  = readme.md
-FILES_INCLUDEMD = summary.md
+FILES_TRANSFORM  = readme.md
+FILES_MAKEDEPS   = readme.md
+FILES_INCLUDEMD1 = summary.md
+FILES_INCLUDEMD2 = readme.md
 
 LUA_REWRITELINKS = ../hugo_rewritelinks.lua
 LUA_MAKEDEPS     = ../hugo_makedeps.lua
@@ -32,9 +33,11 @@ test_makedeps: $(FILES_MAKEDEPS)
 	@$(PANDOC) -L $(LUA_MAKEDEPS) -M prefix="foobar" -M indexMD="readme" -t native $^     \
 	    | $(DIFF) expected_makedeps4.native -
 
-test_includemd: $(FILES_INCLUDEMD)
-	@$(PANDOC) -L $(LUA_INCLUDEMD) -t native $^                                           \
-	    | $(DIFF) expected_inludemd.native -
+test_includemd: $(FILES_INCLUDEMD1) $(FILES_INCLUDEMD2)
+	@$(PANDOC) -L $(LUA_INCLUDEMD) -t native $(FILES_INCLUDEMD1)                          \
+	    | $(DIFF) expected_inludemd1.native -
+	@$(PANDOC) -L $(LUA_INCLUDEMD) -t native $(FILES_INCLUDEMD2)                          \
+	    | $(DIFF) expected_inludemd2.native -
 
 
 expected: expected_rewritelinks1.native expected_rewritelinks2.native expected_rewritelinks3.native expected_rewritelinks4.native
@@ -57,14 +60,16 @@ expected_makedeps3.native: $(FILES_MAKEDEPS)
 expected_makedeps4.native: $(FILES_MAKEDEPS)
 	$(PANDOC) -L $(LUA_MAKEDEPS) -M prefix="foobar" -M indexMD="readme" -t native -o $@ $^
 
-expected: expected_inludemd.native
-expected_inludemd.native: $(FILES_INCLUDEMD)
+expected: expected_inludemd1.native expected_inludemd2.native
+expected_inludemd1.native: $(FILES_INCLUDEMD1)
+	$(PANDOC) -L $(LUA_INCLUDEMD) -t native -o $@ $^
+expected_inludemd2.native: $(FILES_INCLUDEMD2)
 	$(PANDOC) -L $(LUA_INCLUDEMD) -t native -o $@ $^
 
 
 clean:
 	rm -rf expected_rewritelinks1.native expected_rewritelinks2.native expected_rewritelinks3.native expected_rewritelinks4.native
 	rm -rf expected_makedeps1.native expected_makedeps2.native expected_makedeps3.native expected_makedeps4.native
-	rm -rf expected_inludemd.native
+	rm -rf expected_inludemd1.native expected_inludemd2.native
 
 .PHONY: test test_rewritelinks test_makedeps test_includemd expected clean