From 7f6c6f38c8ce9acaae07cb11627ad5b344d2d983 Mon Sep 17 00:00:00 2001 From: jclavier44 Date: Sun, 27 Oct 2024 16:39:55 +0100 Subject: [PATCH] Add the AST json generation feature for external analysis purpose --- docs/compiler_options.md | 1 + tl | 109 ++++++++++++++++++++++++++++++++++----- 2 files changed, 96 insertions(+), 14 deletions(-) diff --git a/docs/compiler_options.md b/docs/compiler_options.md index 0f94e28fa..a6ec6133f 100644 --- a/docs/compiler_options.md +++ b/docs/compiler_options.md @@ -24,6 +24,7 @@ return { | `-I --include-dir` | `include_dir` | `{string}` | `check` `gen` `run` | Prepend this directory to the module search path. | `--gen-compat` | `gen_compat` | `string` | `gen` `run` | Generate compatibility code for targeting different Lua VM versions. See [below](#generated-code) for details. | `--gen-target` | `gen_target` | `string` | `gen` `run` | Minimum targeted Lua version for generated code. Options are `5.1`, `5.3` and `5.4`. See [below](#generated-code) for details. +| | `gen_ast` | `number` | `gen` `run` | generate the AST json file for specific analysis purpose, the number expected is the max extraction depth from AST (use 10 by default, or increase this number for deeper AST info to extract) | `--keep-hashbang` | | | `gen` | Preserve hashbang line (`#!`) at the top of file if present. | `-p --pretend` | | | `gen` | Don't compile/write to any files, but type check and log what files would be written to. | `--wdisable` | `disable_warnings` | `{string}` | `check` `run` | Disable the given warnings. diff --git a/tl b/tl index 9d7322d2b..c3acb4c79 100755 --- a/tl +++ b/tl @@ -139,6 +139,58 @@ local function filename_to_module_name(filename) return (filename:gsub("%.lua$", ""):gsub("%.d%.tl$", ""):gsub("%.tl$", ""):gsub("[/\\]", ".")) end +local function serializeAstTableToJSON(tbl, maxDepth) + local function escapeString(str) + return string.gsub(str, '[%c"\\]', { + ['\t'] = '\\t', + ['\r'] = '\\r', + ['\n'] = '\\n', + ['"'] = '\\"', + ['\\'] = '\\\\' + }) + end + maxDepth = maxDepth or 10 -- Default max depth + + local function serialize(val, stack, depth) + if depth > maxDepth then + return '"MAX_DEPTH_REACHED"' + end + + local t = type(val) + if t == 'string' then + return '"' .. escapeString(val) .. '"' + elseif t == 'number' or t == 'boolean' or t == 'nil' then + return tostring(val) + elseif t == 'table' then + if stack[val] then + return '"CIRCULAR_REF_' .. tostring(stack[val]) .. '"' + end + stack[val] = depth + + local result = {} + local isArray = #val > 0 + local opening, closing = "{", "}" + if isArray then + opening, closing = "[", "]" + for i, v in ipairs(val) do + table.insert(result, serialize(v, stack, depth + 1)) + end + else + for k, v in pairs(val) do + if type(k) ~= 'string' then k = tostring(k) end + table.insert(result, '"' .. escapeString(k) .. '":' .. serialize(v, stack, depth + 1)) + end + end + stack[val] = nil + return opening .. table.concat(result, ",") .. closing + else + return '"UNSUPPORTED_TYPE"' + end + end + + return serialize(tbl, {}, 1) +end + -------------------------------------------------------------------------------- -- Common driver backend -------------------------------------------------------------------------------- @@ -168,6 +220,7 @@ local function setup_env(tlconfig, filename) feat_arity = tlconfig["feat_arity"], gen_compat = tlconfig["gen_compat"], gen_target = tlconfig["gen_target"], + gen_ast = tlconfig["gen_ast"], }, predefined_modules = tlconfig._init_env_modules, } @@ -268,28 +321,50 @@ local function type_check_and_load(tlconfig, filename) return chunk end -local function write_out(tlconfig, result, output_file, pp_opts) +local function write_out(tlconfig, result, output_files, pp_opts) if tlconfig["pretend"] then - print("Would Write: " .. output_file) + print("Would write lua file: " .. output_files[1]) + if tlconfig["gen_ast"]>0 then + print("Would write ast file: " .. output_files[2]) + end return end - local ofd, err = io.open(output_file, "wb") + local ofd, err = io.open(output_files[1], "wb") if not ofd then - die("cannot write " .. output_file .. ": " .. err) + die("cannot write " .. output_files[1] .. ": " .. err) end local _ _, err = ofd:write(tl.generate(result.ast, tlconfig.gen_target, pp_opts) .. "\n") if err then - die("error writing " .. output_file .. ": " .. err) + die("error writing " .. output_files[1] .. ": " .. err) end - ofd:close() if not tlconfig["quiet"] then - print("Wrote: " .. output_file) + print("Wrote: " .. output_files[1]) + end + + if tlconfig["gen_ast"]>0 then + local ofd_ast,err = io.open(output_files[2], "w") + if not ofd_ast then + die("cannot write " .. output_files[2] .. ": " .. err) + else + print("Write ast file: " .. output_files[2]) + _, err =ofd_ast:write('{"ast":\n') + _, err =ofd_ast:write(serializeAstTableToJSON(result.ast,tlconfig["gen_ast"])) + _, err =ofd_ast:write("\n}") + + if err then + die("error writing " .. output_files[2] .. ": " .. err) + end + ofd_ast:close() + if not tlconfig["quiet"] then + print("Wrote: " .. output_files[2]) + end + end end end @@ -331,6 +406,7 @@ local function validate_config(config) gen_target = { ["5.1"] = true, ["5.3"] = true, ["5.4"] = true }, disable_warnings = "{string}", warning_error = "{string}", + gen_ast = "number", } for k, v in pairs(config) do @@ -408,6 +484,8 @@ local function get_args_parser() parser:flag("-p --pretend", "Do not write to any files, type check and output what files would be generated.") + parser:flag("-a --gen-ast", "AST file will be generated.") + parser:require_command(false) parser:command_target("command") @@ -443,7 +521,8 @@ local function get_config(cmd) include_dir = {}, disable_warnings = {}, warning_error = {}, - quiet = false + quiet = false, + gen_ast = 0 } local config_path = find_file_in_parent_dirs("tlconfig.lua") or "tlconfig.lua" @@ -530,6 +609,8 @@ local function merge_config_and_args(tlconfig, args) tlconfig["pretend"] = true end + tlconfig["gen_ast"] = args["gen_ast"] or tlconfig["gen_ast"] + tlconfig["feat_arity"] = args["feat_arity"] or tlconfig["feat_arity"] tlconfig["gen_target"] = args["gen_target"] or tlconfig["gen_target"] @@ -548,9 +629,9 @@ local function get_output_filename(file_name) local name, ext = tail:match("(.+)%.([a-zA-Z]+)$") if not name then name = tail end if ext ~= "lua" then - return name .. ".lua" + return {name .. ".lua",name .. "_ast.json"} else - return name .. ".out.lua" + return {name .. ".out.lua",name .. "_out_ast.json"} end end @@ -750,7 +831,7 @@ commands["check"] = function(tlconfig, args) if ok and tlconfig["quiet"] == false and #args["file"] == 1 then local file_name = args["file"][1] - local output_file = get_output_filename(file_name) + local output_files = get_output_filename(file_name) print("========================================") print("Type checked " .. file_name) print("0 errors detected -- you can use:") @@ -761,7 +842,7 @@ commands["check"] = function(tlconfig, args) print() print(" tl gen " .. file_name) print() - print(" to generate " .. output_file) + print(" to generate " .. output_files[1] .. " and " .. output_files[2]) end os.exit(ok and 0 or 1) @@ -794,7 +875,7 @@ commands["gen"] = function(tlconfig, args) local res = { input_file = input_file, - output_file = get_output_filename(input_file) + output_files = get_output_filename(input_file) } res.tl_result, err = process_module(input_file, env) @@ -808,7 +889,7 @@ commands["gen"] = function(tlconfig, args) for _, res in ipairs(results) do if #res.tl_result.syntax_errors == 0 then - write_out(tlconfig, res.tl_result, args["output"] or res.output_file, pp_opts) + write_out(tlconfig, res.tl_result, args["output"] or res.output_files, pp_opts) end end