fix(yakshadmp): save metadata for macro, and write comment to generat…

…ed json

compiler/libs/libs/c.yaka

@@ -21,8 +21,8 @@ macros! {
             (ykt_string (+ "Const[" yk_import_ref ".Char]")) (ykt_comma)
             (ykt_string (+ "'" (remove_first_last (repr elem::value)) "'")) (ykt_paren_close)))
     (yk_register {dsl const_str const_str})
-    (yk_register {dsl cstr cstr})
     # c.const_str!{"Hi"} --> create a const char * const --> inlinec("Const[Ptr[Const[Char]]]", "\"Hi\"")
+    (yk_register {dsl cstr cstr})
     # c.cstr!{"Hi"} --> create a c.CStr
     (yk_register {dsl char character})
     # c.char!{"a"} --> create a char --> inlinec("Const[Char]", "'a'")

compiler/libs/libs/unittest.yaka

@@ -92,8 +92,26 @@ macros! {
         (list (ykt_keyword_return) (yk_create_token YK_TOKEN_NAME "$x") (ykt_newline) (ykt_dedent))
     ))
     (yk_register {dsl assert_true assert_true})
+    # unittest.assert_true!{"description" x} - assert that this is true
     (yk_register {dsl assert_false assert_false})
+    # unittest.assert_false!{"description" x} - assert that this is false
     (yk_register {dsl test_case test_case})
+    # unittest.test_case!{"name"}: - start a test case definition
+    #
+    # # --- Ensure you use below imports as is ---
+    #
+    # import libs.unittest as u
+    # import libs.console as console
+    #
+    # u.test_case!{"assumptions"}:
+    #     u.assert_true!{"char is 1 byte" (1 == inlinec("int", "sizeof(char)"))}
+    #     x = 1
+    #     u.assert_true!{"int is 4 byte" (4 == inlinec("int", "sizeof(yy__x)"))}
+    #     u.end_test_case!{}
+    #
+    # u.run_all!{}
     (yk_register {dsl end_test_case end_test_case})
+    # see unittest.test_case example above
     (yk_register {dsl run_all run_all})
+    # unittest.run_all!{} - put this at the end to run all unit tests you created
 }

compiler/scripts/libdocs.py

@@ -142,10 +142,13 @@ def cleanup_structure(structure: dict, mod_name: str) -> dict:
         [cleanup_parameter(x, mod_name, imports) for x in structure["global_consts"]], key=lambda x: x["name"])
     structure["classes"] = sorted([cleanup_class(x, mod_name, imports) for x in structure["classes"]],
                                   key=lambda x: x["name"])
-    macro_env = sorted([x["name"] for x in structure["macro_env"]])
+    macro_env = sorted([(x["name"], x) for x in structure["macro_env"]], key=lambda x: x[0])
     del structure["macro_env"]
     dsl_prefix = "yaksha#macro#dsl#"
-    structure["macros"] = [x[len(dsl_prefix):] + "!" for x in macro_env if x.startswith(dsl_prefix)]
+    meta_prefix = "metadata#" + dsl_prefix
+    macro_comments = { x[0][len(meta_prefix):] : x[1]["comment"] for x in macro_env if x[0].startswith(meta_prefix)}
+    macros = [ x[0][len(dsl_prefix):] for x in macro_env if x[0].startswith(dsl_prefix)]
+    structure["macros"] = [{"name": x + "!", "comment": macro_comments.get(x, "") } for x in macros ]
 
     return structure
 
@@ -223,7 +226,7 @@ def display_comment(buf: Buf, element: dict):
         comment_lines = comment.splitlines(keepends=False)
         for single_comment in comment_lines:
             buf.append('\n')
-            buf.append_yellow("# ")
+            buf.append_yellow("#")
             buf.append_yellow(single_comment)
 
 
@@ -232,9 +235,9 @@ def display_param(buf: Buf, param: dict):
     buf.append_red(": ")
     display_datatype(buf, param["datatype"])
 
-def display_mac(buf: Buf, mac: str):
+def display_mac(buf: Buf, mac: dict):
     buf.append_green("macro ")
-    buf.append_cyan(mac)
+    buf.append_cyan(mac["name"])
 
 def display_function(buf: Buf, fnc: dict):
     buf.append_green("def ")
@@ -301,6 +304,7 @@ def main():
         for f in structures[yaksha_mod]["macros"]:
             buf = Buf()
             display_mac(buf, f)
+            display_comment(buf, f)
             print(buf.build_color())
 
         for f in structures[yaksha_mod]["global_consts"]:

compiler/src/dump.cpp

@@ -70,7 +70,7 @@ std::string extract_comments(int definition_line, tokenizer &token_extractor) {
       } else {
         comments << "\n";
       }
-      comments << trim_copy(token->token_);
+      comments << rtrim_copy(token->token_);
     } else {
       break;
     }
@@ -257,7 +257,25 @@ void display(def_class_visitor &df, parser &parser_object,
     }
     std::cout << "{\"name\": \"" << string_utils::escape_json(macro.first)
               << "\", \"type\": \""
-              << yaksha_lisp_value_type_to_str(macro.second->type_) << "\" }";
+              << yaksha_lisp_value_type_to_str(macro.second->type_) << "\"";
+    if (macro.first.rfind("metadata#", 0) == 0) {
+      std::stringstream metadata{};
+      metadata << macro.second;
+      std::cout << ", \"meta\": \"" << string_utils::escape_json(metadata.str())
+                << "\"";
+      if (macro.second->type_ == yaksha_lisp_value_type::LIST &&
+          macro.second->list_.size() == 3) {
+        auto first_expr = macro.second->list_[0];
+        if (first_expr->type_ == yaksha_lisp_value_type::EXPR &&
+            first_expr->expr_->type_ == yaksha_lisp_expr_type::TERMINAL) {
+          auto tok = first_expr->expr_->token_;
+          auto comment = extract_comments(tok->line_, token_extractor);
+          std::cout << ", \"comment\": \"" <<
+            string_utils::escape_json(comment) << "\"";
+        }
+      }
+    }
+    std::cout << " }";
   }
   std::cout << "],";
   // Dump imports

compiler/src/utilities/cpp_util.h

@@ -159,6 +159,10 @@ namespace yaksha {
     trim(s);
     return s;
   }
+  static inline std::string rtrim_copy(std::string s) {
+    rtrim(s);
+    return s;
+  }
   // https://stackoverflow.com/questions/5878775/how-to-find-and-replace-string
   static inline void replace_all(std::string &s, std::string const &to_replace,
                                  std::string const &replace_with) {

compiler/src/yaksha_lisp/prelude.h

@@ -63,7 +63,9 @@ const std::string YAKSHA_LISP_PRELUDE = R"<><><><>(
         (def macro_name (repr (index type_name_func 1)))
         (def macro_func (eval (index type_name_func 2)))
         (def target (+ "yaksha#macro#" macro_type "#" macro_name))
+        (def macro_metadata (+ "metadata#" target))
         (map_set (parent) target macro_func)
+        (map_set (parent) macro_metadata type_name_func)
     )
     # =========== #
     #  Constants

compiler/src/yaksha_lisp/prelude_template.yaka

@@ -22,7 +22,9 @@ macros! {
         (def macro_name (repr (index type_name_func 1)))
         (def macro_func (eval (index type_name_func 2)))
         (def target (+ "yaksha#macro#" macro_type "#" macro_name))
+        (def macro_metadata (+ "metadata#" target))
         (map_set (parent) target macro_func)
+        (map_set (parent) macro_metadata type_name_func)
     )
     # =========== #
     #  Constants