From 0c433a6b4f4ec985c4a413574be02ee24a5c2508 Mon Sep 17 00:00:00 2001 From: JaDogg Date: Sat, 13 Jul 2024 18:37:55 +0100 Subject: [PATCH] feat(parser,ast): decl statements for token replacement --- compiler/scripts/update_ast.py | 26 ++-- compiler/scripts/update_tokens.py | 5 +- compiler/src/ast/ast.cpp | 51 ++++++++ compiler/src/ast/ast.h | 51 ++++++++ compiler/src/ast/parser.cpp | 120 ++++++++++++++++++ compiler/src/ast/parser.h | 1 + compiler/src/compiler/codegen_json.cpp | 6 +- compiler/src/tokenizer/token.h | 92 ++++++++------ compiler/src/yaksha_lisp/prelude.h | 44 ++++--- .../src/yaksha_lisp/prelude_token_gen.yaka | 1 + compiler/src/yaksha_lisp/prelude_tokens.yaka | 43 ++++--- .../directives_and_decl_macors.yaka | 40 ++++++ .../directives_and_decl_macors.yaka.c | 39 ++++++ compiler/tests/test_compiler.cpp | 4 + 14 files changed, 425 insertions(+), 98 deletions(-) create mode 100644 compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka create mode 100644 compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka.c diff --git a/compiler/scripts/update_ast.py b/compiler/scripts/update_ast.py index 17f0356f..bcad128c 100755 --- a/compiler/scripts/update_ast.py +++ b/compiler/scripts/update_ast.py @@ -70,7 +70,7 @@ # Assign to a variable # We can promote an assignment to a let statement, if so promoted is set to true ("assign", (("token*", "name"), ("token*", "opr"), ("expr*", "right"), ("bool", "promoted"), - ("ykdatatype*", "promoted_data_type"))), + ("yk_datatype*", "promoted_data_type"))), # Assign to a member ("assign_member", (("expr*", "set_oper"), ("token*", "opr"), ("expr*", "right"))), # Assign to array object @@ -138,14 +138,18 @@ "import": "import_token_", "runtimefeature": "runtimefeature_token_", "compins": "name_", + "decl": "name_", } -IGNORE_VISITS_STMT = {"elif", "macros", "token_soup", "dsl_macro"} +IGNORE_VISITS_STMT = {"elif", "macros", "token_soup", "dsl_macro", "decl"} # Different kinds of statements STMTS = sorted([ # Directives # directive (os="windows|linux")? (defines="X=Y|A=B")? (no_runtime|no_main|c_include_path|c_link_path|c_link|c_file|ccode) ("STR")? ("directive", (("token*", "directive_token"), ("std::vector", "values"), ("token*", "directive_type"), ("token*", "directive_val"))), + # Declare + # decl red console.red + ("decl", (("token*", "decl_token"), ("token*", "name"), ("std::vector", "replacement"),)), # Define macros in current file. # Parser should check that for a single invoke of parse(), there must be only one, macros section. # -- macros code will be validated and parsed same way. @@ -161,7 +165,7 @@ # Why? # this allows us to store arbitrary tokens in order in statement list ("token_soup", (("std::vector", "soup"),)), - ("return", (("token*", "return_keyword"), ("expr*", "expression"), ("ykdatatype*", "result_type"))), + ("return", (("token*", "return_keyword"), ("expr*", "expression"), ("yk_datatype*", "result_type"))), # defer statement works just like how we use string, deletions. ("defer", (("token*", "defer_keyword"), ("expr*", "expression"), ("stmt*", "del_statement"))), # del statement @@ -183,9 +187,9 @@ # ## or # for x in expr: # println(x) - ("foreach", (("token*", "for_keyword"), ("token*", "name"), ("ykdatatype*", "data_type"), + ("foreach", (("token*", "for_keyword"), ("token*", "name"), ("yk_datatype*", "data_type"), ("token*", "in_keyword"), ("expr*", "expression"), ("stmt*", "for_body"), - ("ykdatatype*", "expr_datatype"),)), + ("yk_datatype*", "expr_datatype"),)), # For loop - endless loop # for: # println("endless") @@ -209,17 +213,17 @@ ("continue", (("token*", "continue_token"),)), ("break", (("token*", "break_token"),)), # Let statements - ("let", (("token*", "name"), ("ykdatatype*", "data_type"), ("expr*", "expression"))), - ("const", (("token*", "name"), ("ykdatatype*", "data_type"), ("expr*", "expression"), ("bool", "is_global"))), + ("let", (("token*", "name"), ("yk_datatype*", "data_type"), ("expr*", "expression"))), + ("const", (("token*", "name"), ("yk_datatype*", "data_type"), ("expr*", "expression"), ("bool", "is_global"))), # Native constant statement # `ITEM: Const[int] = ccode """1 + 1"""` - ("nativeconst", (("token*", "name"), ("ykdatatype*", "data_type"), + ("nativeconst", (("token*", "name"), ("yk_datatype*", "data_type"), ("token*", "ccode_keyword"), ("token*", "code_str"), ("bool", "is_global"))), # Function declarations # Make sure we always say the return type # `def abc(a: int) -> None:` ("def", (("token*", "name"), ("std::vector", "params"), - ("stmt*", "function_body"), ("ykdatatype*", "return_type"), ("annotations", "annotations"))), + ("stmt*", "function_body"), ("yk_datatype*", "return_type"), ("annotations", "annotations"))), ("class", (("token*", "name"), ("std::vector", "members"), ("annotations", "annotations"))), ("enum", (("token*", "name"), ("std::vector", "members"), ("annotations", "annotations"))), # import io [as io] @@ -230,7 +234,7 @@ ("runtimefeature", (("token*", "runtimefeature_token"), ("token*", "feature"))), # ------------ Hidden special instructions for compiler -------------- ("compins", ( - ("token*", "name"), ("ykdatatype*", "data_type"), ("token*", "meta1"), ("ykdatatype*", "meta2"), + ("token*", "name"), ("yk_datatype*", "data_type"), ("token*", "meta1"), ("yk_datatype*", "meta2"), ("void*", "meta3")) ) ], key=lambda x: x[0]) @@ -364,7 +368,7 @@ */ struct parameter { token* name_; - ykdatatype* data_type_; + yk_datatype* data_type_; token* enum_val_override_; }; /** diff --git a/compiler/scripts/update_tokens.py b/compiler/scripts/update_tokens.py index 854763b0..83a66061 100755 --- a/compiler/scripts/update_tokens.py +++ b/compiler/scripts/update_tokens.py @@ -60,7 +60,10 @@ "and", "continue", "for", "try", "as", "def", "from", "while", "assert", "del", "not", - "elif", "if", "or", "defer", "ccode", "runtimefeature", "in", "struct", "macros", "directive", "enum"]) + "elif", "if", "or", "defer", + "ccode", "runtimefeature", "in", + "struct", "macros", "directive", + "enum", "decl"]) TOKENS = sorted([ "NAME", "AT", "DOUBLE_NUMBER", "FLOAT_NUMBER", "INDENT", "BA_INDENT", "BA_DEDENT", "NEW_LINE", "COLON", "SEMICOLON", "COMMENT", diff --git a/compiler/src/ast/ast.cpp b/compiler/src/ast/ast.cpp index 7702fb53..9083fe7a 100644 --- a/compiler/src/ast/ast.cpp +++ b/compiler/src/ast/ast.cpp @@ -1,3 +1,41 @@ +// ============================================================================================== +// ╦ ┬┌─┐┌─┐┌┐┌┌─┐┌─┐ Yaksha Programming Language +// ║ ││ ├┤ │││└─┐├┤ is Licensed with GPLv3 + extra terms. Please see below. +// ╩═╝┴└─┘└─┘┘└┘└─┘└─┘ +// Note: libs - MIT license, runtime/3rd - various +// ============================================================================================== +// GPLv3: +// +// Yaksha - Programming Language. +// Copyright (C) 2020 - 2024 Bhathiya Perera +// +// This program is free software: you can redistribute it and/or modify it under the terms +// of the GNU General Public License as published by the Free Software Foundation, +// either version 3 of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along with this program. +// If not, see https://www.gnu.org/licenses/. +// +// ============================================================================================== +// Additional Terms: +// +// Please note that any commercial use of the programming language's compiler source code +// (everything except compiler/runtime, compiler/libs and compiler/3rd) require a written agreement +// with author of the language (Bhathiya Perera). +// +// If you are using it for an open source project, please give credits. +// Your own project must use GPLv3 license with these additional terms. +// +// You may use programs written in Yaksha/YakshaLisp for any legal purpose +// (commercial, open-source, closed-source, etc) as long as it agrees +// to the licenses of linked runtime libraries (see compiler/runtime/README.md). +// +// ============================================================================================== // ast.cpp // generated by update_ast.py #include "ast/ast.h" @@ -513,6 +551,19 @@ stmt *ast_pool::c_while_stmt(token *while_keyword, expr *expression, cleanup_stmt_.push_back(o); return o; } +decl_stmt::decl_stmt(token *decl_token, token *name, + std::vector replacement) + : decl_token_(decl_token), name_(name), + replacement_(std::move(replacement)) {} +ast_type decl_stmt::get_type() { return ast_type::STMT_DECL; } +void decl_stmt::accept(stmt_visitor *v) {} +token *decl_stmt::locate() { return name_; } +stmt *ast_pool::c_decl_stmt(token *decl_token, token *name, + std::vector replacement) { + auto o = new decl_stmt(decl_token, name, std::move(replacement)); + cleanup_stmt_.push_back(o); + return o; +} dsl_macro_stmt::dsl_macro_stmt(token *name, token *name2, token *not_symbol_tok, token *curly_open, std::vector internal_soup, diff --git a/compiler/src/ast/ast.h b/compiler/src/ast/ast.h index 5cb6decc..b9e68439 100644 --- a/compiler/src/ast/ast.h +++ b/compiler/src/ast/ast.h @@ -1,3 +1,41 @@ +// ============================================================================================== +// ╦ ┬┌─┐┌─┐┌┐┌┌─┐┌─┐ Yaksha Programming Language +// ║ ││ ├┤ │││└─┐├┤ is Licensed with GPLv3 + extra terms. Please see below. +// ╩═╝┴└─┘└─┘┘└┘└─┘└─┘ +// Note: libs - MIT license, runtime/3rd - various +// ============================================================================================== +// GPLv3: +// +// Yaksha - Programming Language. +// Copyright (C) 2020 - 2024 Bhathiya Perera +// +// This program is free software: you can redistribute it and/or modify it under the terms +// of the GNU General Public License as published by the Free Software Foundation, +// either version 3 of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along with this program. +// If not, see https://www.gnu.org/licenses/. +// +// ============================================================================================== +// Additional Terms: +// +// Please note that any commercial use of the programming language's compiler source code +// (everything except compiler/runtime, compiler/libs and compiler/3rd) require a written agreement +// with author of the language (Bhathiya Perera). +// +// If you are using it for an open source project, please give credits. +// Your own project must use GPLv3 license with these additional terms. +// +// You may use programs written in Yaksha/YakshaLisp for any legal purpose +// (commercial, open-source, closed-source, etc) as long as it agrees +// to the licenses of linked runtime libraries (see compiler/runtime/README.md). +// +// ============================================================================================== // ast.h // generated by update_ast.py #ifndef AST_H @@ -36,6 +74,7 @@ namespace yaksha { struct compins_stmt; struct const_stmt; struct continue_stmt; + struct decl_stmt; struct def_stmt; struct defer_stmt; struct del_stmt; @@ -82,6 +121,7 @@ namespace yaksha { STMT_COMPINS, STMT_CONST, STMT_CONTINUE, + STMT_DECL, STMT_DEF, STMT_DEFER, STMT_DEL, @@ -397,6 +437,15 @@ namespace yaksha { token *locate() override; token *continue_token_; }; + struct decl_stmt : stmt { + decl_stmt(token *decl_token, token *name, std::vector replacement); + void accept(stmt_visitor *v) override; + ast_type get_type() override; + token *locate() override; + token *decl_token_; + token *name_; + std::vector replacement_; + }; struct def_stmt : stmt { def_stmt(token *name, std::vector params, stmt *function_body, yk_datatype *return_type, annotations annotations); @@ -637,6 +686,8 @@ namespace yaksha { stmt *c_const_stmt(token *name, yk_datatype *data_type, expr *expression, bool is_global); stmt *c_continue_stmt(token *continue_token); + stmt *c_decl_stmt(token *decl_token, token *name, + std::vector replacement); stmt *c_def_stmt(token *name, std::vector params, stmt *function_body, yk_datatype *return_type, annotations annotations); diff --git a/compiler/src/ast/parser.cpp b/compiler/src/ast/parser.cpp index 3ed58676..dc342efb 100644 --- a/compiler/src/ast/parser.cpp +++ b/compiler/src/ast/parser.cpp @@ -50,6 +50,7 @@ using namespace yaksha; #ifdef YAKSHA_DEBUG #define PRINT_PREPROCESSOR_OUTPUT +#define DUMP_TOKENS_ON_MACRO_ERROR #endif parser::parser(std::string filepath, std::vector &tokens, yk_datatype_pool *pool) @@ -1157,6 +1158,42 @@ void parser::step_4_expand_macros(macro_processor *mp, // Validate tokens for (auto tk : tokens_) { if (!is_valid(tk->token_, tk->type_)) { +#ifdef DUMP_TOKENS_ON_MACRO_ERROR + { + std::cerr << "// Invalid token: [[" << tk->token_ + << "]] type: " << token_to_str(tk->type_) << " at " + << tk->file_ << ":" << tk->line_ << ":" << tk->pos_ + << std::endl; + std::cerr << "// ----------- Preprocessor output for " << filepath_ + << "-------------------" << std::endl; + std::cerr << "/* " << std::endl; + token *prev = nullptr; + for (auto ttk : tokens_) { + // add a space if it is not jammed + if (prev != nullptr && (prev->line_ == ttk->line_) && + (prev->pos_ + prev->token_.size() != ttk->pos_)) { + std::cerr << " "; + } + if (ttk->type_ == token_type::NEW_LINE) { + std::cerr << " [NL]" << std::endl; + } else if (ttk->type_ == token_type::BA_INDENT) { + std::cerr << "`{" << std::endl; + } else if (ttk->type_ == token_type::BA_DEDENT) { + std::cerr << "}`" << std::endl; + } else if (ttk->type_ == token_type::END_OF_FILE) { + std::cerr << "[EOF]" << std::endl; + } else if (ttk->type_ == token_type::STRING) { + std::cerr << "\"" << ttk->token_ << "\""; + } else { + std::cerr << ttk->token_; + } + prev = ttk; + } + std::cerr << "*/" << std::endl; + std::cerr << "// ---------------- end ------------------ " + << std::endl; + } +#endif throw error(tk, "Invalid token"); } } @@ -1165,6 +1202,12 @@ void parser::step_4_expand_macros(macro_processor *mp, errors_.emplace_back(err.message_, err.tok_.file_, err.tok_.line_, err.tok_.pos_); } + // Execute replace decl after macro expansion for second time + if (!errors_.empty()) { return; } + step_replace_decl_alias(); + if (tokens_.empty() || tokens_.back()->type_ != token_type::END_OF_FILE) { + tokens_.emplace_back(original_tokens_.back()); + } } void parser::step_3_execute_macros(macro_processor *mp) { try { @@ -1183,6 +1226,11 @@ void parser::step_3_execute_macros(macro_processor *mp) { } } void parser::step_1_parse_token_soup() { + step_replace_decl_alias(); + if (tokens_.empty() || tokens_.back()->type_ != token_type::END_OF_FILE) { + tokens_.emplace_back(original_tokens_.back()); + } + if (!errors_.empty()) { return; } try { std::vector tokens_buffer{}; parse_token_soup(soup_statements_, tokens_buffer); @@ -1393,4 +1441,76 @@ std::vector parser::parse_enum_members(token *name_token) { consume(token_type::BA_DEDENT, "Expected dedent"); return values; } +void parser::step_replace_decl_alias() { + // TODO verify that alias start in an empty line + try { + // -------- Parse all the DECL statements --------- + std::vector decl_soup{}; + std::vector tokens_buffer{}; + tokens_buffer.reserve(30); + std::unordered_map decls{}; + std::vector new_tokens{}; + new_tokens.reserve(tokens_.size()); + while (!is_at_end_of_stream()) { + // decl red console.red + // decl_keyword alias replacement_tokens + if (match({token_type::KEYWORD_DECL})) { + if (!tokens_buffer.empty()) { + decl_soup.emplace_back(pool_.c_token_soup_stmt(tokens_buffer)); + tokens_buffer.clear(); + } + auto decl_kw = previous(); + auto alias = consume(token_type::NAME, "Alias must be present"); + // get all tokens until newline for replacement + while (!check(token_type::NEW_LINE)) { + tokens_buffer.emplace_back(advance()); + } + consume(token_type::NEW_LINE, "Expect new line after replacement"); + // note we are copying vector, so not a ref + decls[alias->token_] = dynamic_cast( + pool_.c_decl_stmt(decl_kw, alias, tokens_buffer)); + tokens_buffer.clear(); + } else { + tokens_buffer.emplace_back(advance()); + } + } + if (!tokens_buffer.empty()) { + decl_soup.emplace_back(pool_.c_token_soup_stmt(tokens_buffer)); + tokens_buffer.clear(); + } + // --------------- Replace all names that matches with alias ------------ + if (decls.empty()) { + // No decl statements, so no need to replace anything + current_ = 0; + return; + } + for (stmt *soup : decl_soup) { + if (soup->get_type() == ast_type::STMT_TOKEN_SOUP) { + auto tokens = dynamic_cast(soup); + for (token *tk : tokens->soup_) { + if (tk->type_ == token_type::NAME) { + auto decl = decls.find(tk->token_); + if (decl != decls.end()) { + auto replacement = decl->second->replacement_; + for (token *rt : replacement) { + rt->line_ = tk->line_; + rt->pos_ = tk->pos_; + new_tokens.emplace_back(rt); + } + } else { + new_tokens.emplace_back(tk); + } + } else { + new_tokens.emplace_back(tk); + } + } + } + } + current_ = 0; + tokens_ = new_tokens; + } catch (parsing_error &err) { + errors_.emplace_back(err.message_, err.tok_.file_, err.tok_.line_, + err.tok_.pos_); + } +} #pragma clang diagnostic pop diff --git a/compiler/src/ast/parser.h b/compiler/src/ast/parser.h index 494d351a..9a56b365 100644 --- a/compiler/src/ast/parser.h +++ b/compiler/src/ast/parser.h @@ -54,6 +54,7 @@ namespace yaksha { yk_datatype_pool *pool); ~parser(); // stepwise preprocess dsl macros + void step_replace_decl_alias(); void step_1_parse_token_soup(); void step_3_execute_macros(macro_processor *mp); void step_4_expand_macros(macro_processor *mp, gc_pool *token_pool); diff --git a/compiler/src/compiler/codegen_json.cpp b/compiler/src/compiler/codegen_json.cpp index 2a19df69..d4250fd7 100644 --- a/compiler/src/compiler/codegen_json.cpp +++ b/compiler/src/compiler/codegen_json.cpp @@ -445,12 +445,14 @@ void to_json_compiler::visit_enum_stmt(enum_stmt *obj) { json_ << "{\n"; write_location(obj); json_ << "\"type\": \"enum\",\n"; - json_ << "\"name\": \"" << string_utils::escape_json(obj->name_->token_) << "\",\n"; + json_ << "\"name\": \"" << string_utils::escape_json(obj->name_->token_) + << "\",\n"; json_ << "\"values\": [\n"; int index = 0; for (auto &value : obj->members_) { json_ << "{\n"; - json_ << "\"name\": \"" << string_utils::escape_json(value.name_->token_) << "\",\n"; + json_ << "\"name\": \"" << string_utils::escape_json(value.name_->token_) + << "\",\n"; json_ << "\"value\": " << index << "\n"; json_ << "}\n"; if (&value != &obj->members_.back()) { json_ << ","; } diff --git a/compiler/src/tokenizer/token.h b/compiler/src/tokenizer/token.h index ab8dff88..03ac42aa 100644 --- a/compiler/src/tokenizer/token.h +++ b/compiler/src/tokenizer/token.h @@ -113,6 +113,7 @@ namespace yaksha { KEYWORD_CCODE, KEYWORD_CLASS, KEYWORD_CONTINUE, + KEYWORD_DECL, KEYWORD_DEF, KEYWORD_DEFER, KEYWORD_DEL, @@ -149,6 +150,7 @@ namespace yaksha { case token_type::KEYWORD_CCODE: case token_type::KEYWORD_CLASS: case token_type::KEYWORD_CONTINUE: + case token_type::KEYWORD_DECL: case token_type::KEYWORD_DEF: case token_type::KEYWORD_DEFER: case token_type::KEYWORD_DEL: @@ -285,6 +287,7 @@ namespace yaksha { if (t == token_type::KEYWORD_CCODE) return "KEYWORD_CCODE"; if (t == token_type::KEYWORD_CLASS) return "KEYWORD_CLASS"; if (t == token_type::KEYWORD_CONTINUE) return "KEYWORD_CONTINUE"; + if (t == token_type::KEYWORD_DECL) return "KEYWORD_DECL"; if (t == token_type::KEYWORD_DEF) return "KEYWORD_DEF"; if (t == token_type::KEYWORD_DEFER) return "KEYWORD_DEFER"; if (t == token_type::KEYWORD_DEL) return "KEYWORD_DEL"; @@ -413,6 +416,7 @@ namespace yaksha { if (t == "KEYWORD_CCODE") return token_type::KEYWORD_CCODE; if (t == "KEYWORD_CLASS") return token_type::KEYWORD_CLASS; if (t == "KEYWORD_CONTINUE") return token_type::KEYWORD_CONTINUE; + if (t == "KEYWORD_DECL") return token_type::KEYWORD_DECL; if (t == "KEYWORD_DEF") return token_type::KEYWORD_DEF; if (t == "KEYWORD_DEFER") return token_type::KEYWORD_DEFER; if (t == "KEYWORD_DEL") return token_type::KEYWORD_DEL; @@ -448,6 +452,7 @@ namespace yaksha { if (t == "ccode") return token_type::KEYWORD_CCODE; if (t == "class") return token_type::KEYWORD_CLASS; if (t == "continue") return token_type::KEYWORD_CONTINUE; + if (t == "decl") return token_type::KEYWORD_DECL; if (t == "def") return token_type::KEYWORD_DEF; if (t == "defer") return token_type::KEYWORD_DEFER; if (t == "del") return token_type::KEYWORD_DEL; @@ -568,27 +573,28 @@ namespace yaksha { if (n == 28964) return token_type::KEYWORD_CCODE; if (n == 28965) return token_type::KEYWORD_CLASS; if (n == 28966) return token_type::KEYWORD_CONTINUE; - if (n == 28967) return token_type::KEYWORD_DEF; - if (n == 28968) return token_type::KEYWORD_DEFER; - if (n == 28969) return token_type::KEYWORD_DEL; - if (n == 28970) return token_type::KEYWORD_DIRECTIVE; - if (n == 28971) return token_type::KEYWORD_ELIF; - if (n == 28972) return token_type::KEYWORD_ELSE; - if (n == 28973) return token_type::KEYWORD_ENUM; - if (n == 28974) return token_type::KEYWORD_FOR; - if (n == 28975) return token_type::KEYWORD_FROM; - if (n == 28976) return token_type::KEYWORD_IF; - if (n == 28977) return token_type::KEYWORD_IMPORT; - if (n == 28978) return token_type::KEYWORD_IN; - if (n == 28979) return token_type::KEYWORD_MACROS; - if (n == 28980) return token_type::KEYWORD_NOT; - if (n == 28981) return token_type::KEYWORD_OR; - if (n == 28982) return token_type::KEYWORD_PASS; - if (n == 28983) return token_type::KEYWORD_RETURN; - if (n == 28984) return token_type::KEYWORD_RUNTIMEFEATURE; - if (n == 28985) return token_type::KEYWORD_STRUCT; - if (n == 28986) return token_type::KEYWORD_TRY; - if (n == 28987) return token_type::KEYWORD_WHILE; + if (n == 28967) return token_type::KEYWORD_DECL; + if (n == 28968) return token_type::KEYWORD_DEF; + if (n == 28969) return token_type::KEYWORD_DEFER; + if (n == 28970) return token_type::KEYWORD_DEL; + if (n == 28971) return token_type::KEYWORD_DIRECTIVE; + if (n == 28972) return token_type::KEYWORD_ELIF; + if (n == 28973) return token_type::KEYWORD_ELSE; + if (n == 28974) return token_type::KEYWORD_ENUM; + if (n == 28975) return token_type::KEYWORD_FOR; + if (n == 28976) return token_type::KEYWORD_FROM; + if (n == 28977) return token_type::KEYWORD_IF; + if (n == 28978) return token_type::KEYWORD_IMPORT; + if (n == 28979) return token_type::KEYWORD_IN; + if (n == 28980) return token_type::KEYWORD_MACROS; + if (n == 28981) return token_type::KEYWORD_NOT; + if (n == 28982) return token_type::KEYWORD_OR; + if (n == 28983) return token_type::KEYWORD_PASS; + if (n == 28984) return token_type::KEYWORD_RETURN; + if (n == 28985) return token_type::KEYWORD_RUNTIMEFEATURE; + if (n == 28986) return token_type::KEYWORD_STRUCT; + if (n == 28987) return token_type::KEYWORD_TRY; + if (n == 28988) return token_type::KEYWORD_WHILE; return token_type::TK_UNKNOWN_TOKEN_DETECTED; } static inline int64_t token_to_numeric_id(const token_type &t) { @@ -688,27 +694,28 @@ namespace yaksha { if (t == token_type::KEYWORD_CCODE) return 28964; if (t == token_type::KEYWORD_CLASS) return 28965; if (t == token_type::KEYWORD_CONTINUE) return 28966; - if (t == token_type::KEYWORD_DEF) return 28967; - if (t == token_type::KEYWORD_DEFER) return 28968; - if (t == token_type::KEYWORD_DEL) return 28969; - if (t == token_type::KEYWORD_DIRECTIVE) return 28970; - if (t == token_type::KEYWORD_ELIF) return 28971; - if (t == token_type::KEYWORD_ELSE) return 28972; - if (t == token_type::KEYWORD_ENUM) return 28973; - if (t == token_type::KEYWORD_FOR) return 28974; - if (t == token_type::KEYWORD_FROM) return 28975; - if (t == token_type::KEYWORD_IF) return 28976; - if (t == token_type::KEYWORD_IMPORT) return 28977; - if (t == token_type::KEYWORD_IN) return 28978; - if (t == token_type::KEYWORD_MACROS) return 28979; - if (t == token_type::KEYWORD_NOT) return 28980; - if (t == token_type::KEYWORD_OR) return 28981; - if (t == token_type::KEYWORD_PASS) return 28982; - if (t == token_type::KEYWORD_RETURN) return 28983; - if (t == token_type::KEYWORD_RUNTIMEFEATURE) return 28984; - if (t == token_type::KEYWORD_STRUCT) return 28985; - if (t == token_type::KEYWORD_TRY) return 28986; - if (t == token_type::KEYWORD_WHILE) return 28987; + if (t == token_type::KEYWORD_DECL) return 28967; + if (t == token_type::KEYWORD_DEF) return 28968; + if (t == token_type::KEYWORD_DEFER) return 28969; + if (t == token_type::KEYWORD_DEL) return 28970; + if (t == token_type::KEYWORD_DIRECTIVE) return 28971; + if (t == token_type::KEYWORD_ELIF) return 28972; + if (t == token_type::KEYWORD_ELSE) return 28973; + if (t == token_type::KEYWORD_ENUM) return 28974; + if (t == token_type::KEYWORD_FOR) return 28975; + if (t == token_type::KEYWORD_FROM) return 28976; + if (t == token_type::KEYWORD_IF) return 28977; + if (t == token_type::KEYWORD_IMPORT) return 28978; + if (t == token_type::KEYWORD_IN) return 28979; + if (t == token_type::KEYWORD_MACROS) return 28980; + if (t == token_type::KEYWORD_NOT) return 28981; + if (t == token_type::KEYWORD_OR) return 28982; + if (t == token_type::KEYWORD_PASS) return 28983; + if (t == token_type::KEYWORD_RETURN) return 28984; + if (t == token_type::KEYWORD_RUNTIMEFEATURE) return 28985; + if (t == token_type::KEYWORD_STRUCT) return 28986; + if (t == token_type::KEYWORD_TRY) return 28987; + if (t == token_type::KEYWORD_WHILE) return 28988; return 0; } static inline bool is_number_token(const token_type &t) { @@ -889,6 +896,7 @@ namespace yaksha { if (t == token_type::KEYWORD_CCODE) return (value == "ccode"); if (t == token_type::KEYWORD_CLASS) return (value == "class"); if (t == token_type::KEYWORD_CONTINUE) return (value == "continue"); + if (t == token_type::KEYWORD_DECL) return (value == "decl"); if (t == token_type::KEYWORD_DEF) return (value == "def"); if (t == token_type::KEYWORD_DEFER) return (value == "defer"); if (t == token_type::KEYWORD_DEL) return (value == "del"); diff --git a/compiler/src/yaksha_lisp/prelude.h b/compiler/src/yaksha_lisp/prelude.h index d12ec428..ee836d58 100644 --- a/compiler/src/yaksha_lisp/prelude.h +++ b/compiler/src/yaksha_lisp/prelude.h @@ -166,27 +166,28 @@ const std::string YAKSHA_LISP_PRELUDE = R"<><><><>( (def YK_TOKEN_KEYWORD_CCODE 28964) (def YK_TOKEN_KEYWORD_CLASS 28965) (def YK_TOKEN_KEYWORD_CONTINUE 28966) - (def YK_TOKEN_KEYWORD_DEF 28967) - (def YK_TOKEN_KEYWORD_DEFER 28968) - (def YK_TOKEN_KEYWORD_DEL 28969) - (def YK_TOKEN_KEYWORD_DIRECTIVE 28970) - (def YK_TOKEN_KEYWORD_ELIF 28971) - (def YK_TOKEN_KEYWORD_ELSE 28972) - (def YK_TOKEN_KEYWORD_ENUM 28973) - (def YK_TOKEN_KEYWORD_FOR 28974) - (def YK_TOKEN_KEYWORD_FROM 28975) - (def YK_TOKEN_KEYWORD_IF 28976) - (def YK_TOKEN_KEYWORD_IMPORT 28977) - (def YK_TOKEN_KEYWORD_IN 28978) - (def YK_TOKEN_KEYWORD_MACROS 28979) - (def YK_TOKEN_KEYWORD_NOT 28980) - (def YK_TOKEN_KEYWORD_OR 28981) - (def YK_TOKEN_KEYWORD_PASS 28982) - (def YK_TOKEN_KEYWORD_RETURN 28983) - (def YK_TOKEN_KEYWORD_RUNTIMEFEATURE 28984) - (def YK_TOKEN_KEYWORD_STRUCT 28985) - (def YK_TOKEN_KEYWORD_TRY 28986) - (def YK_TOKEN_KEYWORD_WHILE 28987) + (def YK_TOKEN_KEYWORD_DECL 28967) + (def YK_TOKEN_KEYWORD_DEF 28968) + (def YK_TOKEN_KEYWORD_DEFER 28969) + (def YK_TOKEN_KEYWORD_DEL 28970) + (def YK_TOKEN_KEYWORD_DIRECTIVE 28971) + (def YK_TOKEN_KEYWORD_ELIF 28972) + (def YK_TOKEN_KEYWORD_ELSE 28973) + (def YK_TOKEN_KEYWORD_ENUM 28974) + (def YK_TOKEN_KEYWORD_FOR 28975) + (def YK_TOKEN_KEYWORD_FROM 28976) + (def YK_TOKEN_KEYWORD_IF 28977) + (def YK_TOKEN_KEYWORD_IMPORT 28978) + (def YK_TOKEN_KEYWORD_IN 28979) + (def YK_TOKEN_KEYWORD_MACROS 28980) + (def YK_TOKEN_KEYWORD_NOT 28981) + (def YK_TOKEN_KEYWORD_OR 28982) + (def YK_TOKEN_KEYWORD_PASS 28983) + (def YK_TOKEN_KEYWORD_RETURN 28984) + (def YK_TOKEN_KEYWORD_RUNTIMEFEATURE 28985) + (def YK_TOKEN_KEYWORD_STRUCT 28986) + (def YK_TOKEN_KEYWORD_TRY 28987) + (def YK_TOKEN_KEYWORD_WHILE 28988) # =========== # # Generating tokens # =========== # @@ -262,6 +263,7 @@ const std::string YAKSHA_LISP_PRELUDE = R"<><><><>( (defun ykt_keyword_ccode () (yk_create_token YK_TOKEN_KEYWORD_CCODE "ccode")) (defun ykt_keyword_class () (yk_create_token YK_TOKEN_KEYWORD_CLASS "class")) (defun ykt_keyword_continue () (yk_create_token YK_TOKEN_KEYWORD_CONTINUE "continue")) + (defun ykt_keyword_decl () (yk_create_token YK_TOKEN_KEYWORD_DECL "decl")) (defun ykt_keyword_def () (yk_create_token YK_TOKEN_KEYWORD_DEF "def")) (defun ykt_keyword_defer () (yk_create_token YK_TOKEN_KEYWORD_DEFER "defer")) (defun ykt_keyword_del () (yk_create_token YK_TOKEN_KEYWORD_DEL "del")) diff --git a/compiler/src/yaksha_lisp/prelude_token_gen.yaka b/compiler/src/yaksha_lisp/prelude_token_gen.yaka index 185c6862..1d706ee4 100644 --- a/compiler/src/yaksha_lisp/prelude_token_gen.yaka +++ b/compiler/src/yaksha_lisp/prelude_token_gen.yaka @@ -54,6 +54,7 @@ (defun ykt_keyword_ccode () (yk_create_token YK_TOKEN_KEYWORD_CCODE "ccode")) (defun ykt_keyword_class () (yk_create_token YK_TOKEN_KEYWORD_CLASS "class")) (defun ykt_keyword_continue () (yk_create_token YK_TOKEN_KEYWORD_CONTINUE "continue")) + (defun ykt_keyword_decl () (yk_create_token YK_TOKEN_KEYWORD_DECL "decl")) (defun ykt_keyword_def () (yk_create_token YK_TOKEN_KEYWORD_DEF "def")) (defun ykt_keyword_defer () (yk_create_token YK_TOKEN_KEYWORD_DEFER "defer")) (defun ykt_keyword_del () (yk_create_token YK_TOKEN_KEYWORD_DEL "del")) diff --git a/compiler/src/yaksha_lisp/prelude_tokens.yaka b/compiler/src/yaksha_lisp/prelude_tokens.yaka index 025a2ad5..325acea4 100644 --- a/compiler/src/yaksha_lisp/prelude_tokens.yaka +++ b/compiler/src/yaksha_lisp/prelude_tokens.yaka @@ -94,24 +94,25 @@ (def YK_TOKEN_KEYWORD_CCODE 28964) (def YK_TOKEN_KEYWORD_CLASS 28965) (def YK_TOKEN_KEYWORD_CONTINUE 28966) - (def YK_TOKEN_KEYWORD_DEF 28967) - (def YK_TOKEN_KEYWORD_DEFER 28968) - (def YK_TOKEN_KEYWORD_DEL 28969) - (def YK_TOKEN_KEYWORD_DIRECTIVE 28970) - (def YK_TOKEN_KEYWORD_ELIF 28971) - (def YK_TOKEN_KEYWORD_ELSE 28972) - (def YK_TOKEN_KEYWORD_ENUM 28973) - (def YK_TOKEN_KEYWORD_FOR 28974) - (def YK_TOKEN_KEYWORD_FROM 28975) - (def YK_TOKEN_KEYWORD_IF 28976) - (def YK_TOKEN_KEYWORD_IMPORT 28977) - (def YK_TOKEN_KEYWORD_IN 28978) - (def YK_TOKEN_KEYWORD_MACROS 28979) - (def YK_TOKEN_KEYWORD_NOT 28980) - (def YK_TOKEN_KEYWORD_OR 28981) - (def YK_TOKEN_KEYWORD_PASS 28982) - (def YK_TOKEN_KEYWORD_RETURN 28983) - (def YK_TOKEN_KEYWORD_RUNTIMEFEATURE 28984) - (def YK_TOKEN_KEYWORD_STRUCT 28985) - (def YK_TOKEN_KEYWORD_TRY 28986) - (def YK_TOKEN_KEYWORD_WHILE 28987) \ No newline at end of file + (def YK_TOKEN_KEYWORD_DECL 28967) + (def YK_TOKEN_KEYWORD_DEF 28968) + (def YK_TOKEN_KEYWORD_DEFER 28969) + (def YK_TOKEN_KEYWORD_DEL 28970) + (def YK_TOKEN_KEYWORD_DIRECTIVE 28971) + (def YK_TOKEN_KEYWORD_ELIF 28972) + (def YK_TOKEN_KEYWORD_ELSE 28973) + (def YK_TOKEN_KEYWORD_ENUM 28974) + (def YK_TOKEN_KEYWORD_FOR 28975) + (def YK_TOKEN_KEYWORD_FROM 28976) + (def YK_TOKEN_KEYWORD_IF 28977) + (def YK_TOKEN_KEYWORD_IMPORT 28978) + (def YK_TOKEN_KEYWORD_IN 28979) + (def YK_TOKEN_KEYWORD_MACROS 28980) + (def YK_TOKEN_KEYWORD_NOT 28981) + (def YK_TOKEN_KEYWORD_OR 28982) + (def YK_TOKEN_KEYWORD_PASS 28983) + (def YK_TOKEN_KEYWORD_RETURN 28984) + (def YK_TOKEN_KEYWORD_RUNTIMEFEATURE 28985) + (def YK_TOKEN_KEYWORD_STRUCT 28986) + (def YK_TOKEN_KEYWORD_TRY 28987) + (def YK_TOKEN_KEYWORD_WHILE 28988) \ No newline at end of file diff --git a/compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka b/compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka new file mode 100644 index 00000000..024e9be1 --- /dev/null +++ b/compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka @@ -0,0 +1,40 @@ +# Exposing global mutable state using embedded C code using a directive ccode +directive ccode """ +void* GLOBAL_STATE = NULL; + +void* get_global_state() { + return GLOBAL_STATE; +} + +void set_global_state(void* state) { + GLOBAL_STATE = state; +} +""" + +decl s get_global_state() + +class MyState: + x: int + y: int + +@nativedefine("get_global_state") +def get_global_state() -> MyState: + pass + +@nativedefine("set_global_state") +def set_global_state(state: MyState) -> None: + pass + +def main() -> int: + state = MyState() + defer del state + state.x = 1 + state.y = 2 + set_global_state(state) + println(state.x) + println(state.y) + println(s.x) + println(s.y) + state.x = 5 + println(s.x) + 0 diff --git a/compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka.c b/compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka.c new file mode 100644 index 00000000..18830f7d --- /dev/null +++ b/compiler/test_data/compiler_tests/directives/directives_and_decl_macors.yaka.c @@ -0,0 +1,39 @@ +// YK +#include "yk__lib.h" +#define yy__get_global_state get_global_state +#define yy__set_global_state set_global_state +struct yy__MyState; +struct yy__MyState { + int32_t yy__x; + int32_t yy__y; +}; +int32_t yy__main(); + +void* GLOBAL_STATE = NULL; + +void* get_global_state() { + return GLOBAL_STATE; +} + +void set_global_state(void* state) { + GLOBAL_STATE = state; +} +; +int32_t yy__main() +{ + struct yy__MyState* yy__state = calloc(1, sizeof(struct yy__MyState)); + yy__state->yy__x = INT32_C(1); + yy__state->yy__y = INT32_C(2); + yy__set_global_state(yy__state); + yk__printlnint((intmax_t)yy__state->yy__x); + yk__printlnint((intmax_t)yy__state->yy__y); + yk__printlnint((intmax_t)yy__get_global_state()->yy__x); + yk__printlnint((intmax_t)yy__get_global_state()->yy__y); + yy__state->yy__x = INT32_C(5); + yk__printlnint((intmax_t)yy__get_global_state()->yy__x); + free(yy__state); + return INT32_C(0); +} +#if defined(YK__MINIMAL_MAIN) +int main(void) { return yy__main(); } +#endif \ No newline at end of file diff --git a/compiler/tests/test_compiler.cpp b/compiler/tests/test_compiler.cpp index b24afb8f..595c1a6a 100644 --- a/compiler/tests/test_compiler.cpp +++ b/compiler/tests/test_compiler.cpp @@ -398,4 +398,8 @@ TEST_CASE("compiler: structures - depends on other structures") { TEST_CASE("compiler: enums - import and use enum") { test_compile_yaka_file( "../test_data/compiler_tests/integer_enums/sample.yaka"); +} +TEST_CASE("compiler: decl macro - decl s get_global_state()") { + test_compile_yaka_file( + "../test_data/compiler_tests/directives/directives_and_decl_macors.yaka"); } \ No newline at end of file