diff --git a/spec/language_server/ls_spec.cr b/spec/language_server/ls_spec.cr index 78fd357a1..ef9b05518 100644 --- a/spec/language_server/ls_spec.cr +++ b/spec/language_server/ls_spec.cr @@ -5,7 +5,7 @@ def clean_json(workspace : Workspace, path : String) end Dir - .glob("./spec/language_server/{definition,hover}/**/*") + .glob("./spec/language_server/{definition,hover,semantic_tokens}/**/*") .select! { |file| File.file?(file) } .sort! .each do |file| diff --git a/spec/language_server/semantic_tokens/record b/spec/language_server/semantic_tokens/record new file mode 100644 index 000000000..deadce430 --- /dev/null +++ b/spec/language_server/semantic_tokens/record @@ -0,0 +1,87 @@ +/* Some comment. */ +record Article { + id : Number, + description : String, + title : String +} +-----------------------------------------------------------------file test.mint +{ + "id": 0, + "method": "initialize", + "params": { + "capabilities": { + "textDocument": { + "semanticTokens": { + "dynamicRegistration": false, + "tokenTypes": ["property"] + } + } + } + } +} +-------------------------------------------------------------------------request +{ + "jsonrpc": "2.0", + "id": 1, + "params": { + "textDocument": { + "uri": "file://#{root_path}/test.mint" + } + }, + "method": "textDocument/semanticTokens/full" +} +-------------------------------------------------------------------------request +{ + "jsonrpc": "2.0", + "result": { + "data": [ + 0, + 0, + 20, + 5, + 0, + 1, + 0, + 6, + 4, + 0, + 0, + 7, + 7, + 1, + 0, + 1, + 2, + 2, + 6, + 0, + 0, + 5, + 6, + 1, + 0, + 1, + 2, + 11, + 6, + 0, + 0, + 14, + 6, + 1, + 0, + 1, + 2, + 5, + 6, + 0, + 0, + 8, + 6, + 1, + 0 + ] + }, + "id": 1 +} +------------------------------------------------------------------------response diff --git a/src/all.cr b/src/all.cr index de99cefb9..c9f6e48d1 100644 --- a/src/all.cr +++ b/src/all.cr @@ -57,6 +57,8 @@ require "./documentation_generator/**" require "./documentation_generator" require "./documentation_server" +require "./semantic_tokenizer" + require "./test_runner/**" require "./test_runner" diff --git a/src/ast.cr b/src/ast.cr index 9e123e314..539e32c83 100644 --- a/src/ast.cr +++ b/src/ast.cr @@ -31,9 +31,12 @@ module Mint Js getter components, modules, records, stores, routes, providers - getter suites, enums, comments, nodes, unified_modules + getter suites, enums, comments, nodes, unified_modules, keywords + getter operators - def initialize(@records = [] of RecordDefinition, + def initialize(@operators = [] of Tuple(Int32, Int32), + @keywords = [] of Tuple(Int32, Int32), + @records = [] of RecordDefinition, @unified_modules = [] of Module, @components = [] of Component, @providers = [] of Provider, diff --git a/src/ast/directives/highlight.cr b/src/ast/directives/highlight.cr new file mode 100644 index 000000000..3f4ac2ba0 --- /dev/null +++ b/src/ast/directives/highlight.cr @@ -0,0 +1,15 @@ +module Mint + class Ast + module Directives + class Highlight < Node + getter content + + def initialize(@content : Block, + @input : Data, + @from : Int32, + @to : Int32) + end + end + end + end +end diff --git a/src/ast/html_component.cr b/src/ast/html_component.cr index 99fefbdff..f999db33b 100644 --- a/src/ast/html_component.cr +++ b/src/ast/html_component.cr @@ -1,13 +1,14 @@ module Mint class Ast class HtmlComponent < Node - getter attributes, children, component, comments, ref + getter attributes, children, component, comments, ref, closing_tag_position def initialize(@attributes : Array(HtmlAttribute), + @closing_tag_position : Int32?, @comments : Array(Comment), @children : Array(Node), - @ref : Variable?, @component : TypeId, + @ref : Variable?, @input : Data, @from : Int32, @to : Int32) diff --git a/src/ast/html_element.cr b/src/ast/html_element.cr index de4136589..d587353c6 100644 --- a/src/ast/html_element.cr +++ b/src/ast/html_element.cr @@ -2,8 +2,10 @@ module Mint class Ast class HtmlElement < Node getter attributes, children, styles, tag, comments, ref + getter closing_tag_position def initialize(@attributes : Array(HtmlAttribute), + @closing_tag_position : Int32?, @comments : Array(Comment), @styles : Array(HtmlStyle), @children : Array(Node), diff --git a/src/ast/node.cr b/src/ast/node.cr index 725b64d8f..9a1565fa6 100644 --- a/src/ast/node.cr +++ b/src/ast/node.cr @@ -71,7 +71,7 @@ module Mint source.strip.includes?('\n') end - protected def compute_position(lines, needle) : Position + def self.compute_position(lines, needle) : Position line_start_pos, line = begin left, right = 0, lines.size - 1 index = pos = 0 @@ -107,19 +107,23 @@ module Mint {line, column} end - getter location : Location do + def self.compute_location(input : Data, from, to) # TODO: avoid creating this array for every (initial) call to `Node#location` lines = [0] - @input.input.each_char_with_index do |ch, i| + input.input.each_char_with_index do |ch, i| lines << i + 1 if ch == '\n' end Location.new( - filename: @input.file, + filename: input.file, start: compute_position(lines, from), end: compute_position(lines, to), ) end + + getter location : Location do + Node.compute_location(input, from, to) + end end end end diff --git a/src/cli.cr b/src/cli.cr index 050e9f48f..dad26230e 100644 --- a/src/cli.cr +++ b/src/cli.cr @@ -15,6 +15,7 @@ module Mint define_help description: "Mint" register_sub_command "sandbox-server", type: SandboxServer + register_sub_command highlight, type: Highlight register_sub_command install, type: Install register_sub_command compile, type: Compile register_sub_command version, type: Version diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr new file mode 100644 index 000000000..6b1195f0a --- /dev/null +++ b/src/commands/highlight.cr @@ -0,0 +1,20 @@ +module Mint + class Cli < Admiral::Command + class Highlight < Admiral::Command + include Command + + define_help description: "Returns the syntax highlighted version of the given file" + + define_argument path, description: "The path to the file" + + define_flag html : Bool, + description: "If specified, print the highlighted code as HTML", + default: false + + def run + return unless path = arguments.path + puts SemanticTokenizer.highlight(path, flags.html) + end + end + end +end diff --git a/src/compilers/directives/highlight.cr b/src/compilers/directives/highlight.cr new file mode 100644 index 000000000..8690696a9 --- /dev/null +++ b/src/compilers/directives/highlight.cr @@ -0,0 +1,36 @@ +module Mint + class Compiler + def _compile(node : Ast::Directives::Highlight) : String + content = + compile node.content + + formatted = + Formatter.new.format(node.content, Formatter::BlockFormat::Naked) + + parser = Parser.new(formatted, "source.mint") + parser.code_block_naked + + parts = + SemanticTokenizer.tokenize(parser.ast) + + mapped = + parts.map do |item| + case item + in String + "`#{skip { escape_for_javascript(item) }}`" + in Tuple(SemanticTokenizer::TokenType, String) + "_h('span', { className: '#{item[0].to_s.underscore}' }, [`#{skip { escape_for_javascript(item[1]) }}`])" + end + end + + "[#{content}, _h(React.Fragment, {}, [#{mapped.join(",\n")}])]" + end + + def escape_for_javascript(value : String) + value + .gsub('\\', "\\\\") + .gsub('`', "\\`") + .gsub("${", "\\${") + end + end +end diff --git a/src/formatters/directives/highlight.cr b/src/formatters/directives/highlight.cr new file mode 100644 index 000000000..ba0408468 --- /dev/null +++ b/src/formatters/directives/highlight.cr @@ -0,0 +1,7 @@ +module Mint + class Formatter + def format(node : Ast::Directives::Highlight) + "@highlight #{format(node.content)}" + end + end +end diff --git a/src/ls/definition/type_id.cr b/src/ls/definition/type_id.cr new file mode 100644 index 000000000..eed5a2a24 --- /dev/null +++ b/src/ls/definition/type_id.cr @@ -0,0 +1,24 @@ +module Mint + module LS + class Definition < LSP::RequestMessage + def definition(node : Ast::TypeId, server : Server, workspace : Workspace, stack : Array(Ast::Node)) + found = + workspace.ast.enums.find(&.name.value.==(node.value)) || + workspace.ast.records.find(&.name.value.==(node.value)) || + workspace.ast.stores.find(&.name.value.==(node.value)) || + find_component(workspace, node.value) + + if found.nil? && (next_node = stack[1]) + return definition(next_node, server, workspace, stack) + end + + return if Core.ast.nodes.includes?(found) + + case found + when Ast::Store, Ast::Enum, Ast::Component, Ast::RecordDefinition + location_link server, node, found.name, found + end + end + end + end +end diff --git a/src/ls/initialize.cr b/src/ls/initialize.cr index 0d7305fff..4a712d096 100644 --- a/src/ls/initialize.cr +++ b/src/ls/initialize.cr @@ -42,10 +42,20 @@ module Mint change_notifications: false, supported: false)) + semantic_tokens_provider = + LSP::SemanticTokensOptions.new( + range: false, + full: true, + legend: LSP::SemanticTokensLegend.new( + token_types: SemanticTokenizer::TOKEN_TYPES, + token_modifiers: [] of String, + )) + capabilities = LSP::ServerCapabilities.new( document_on_type_formatting_provider: document_on_type_formatting_provider, execute_command_provider: execute_command_provider, + semantic_tokens_provider: semantic_tokens_provider, signature_help_provider: signature_help_provider, document_link_provider: document_link_provider, completion_provider: completion_provider, diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr new file mode 100644 index 000000000..aeb24a56d --- /dev/null +++ b/src/ls/semantic_tokens.cr @@ -0,0 +1,65 @@ +module Mint + module LS + # This is the class that handles the "textDocument/semanticTokens/full" request. + class SemanticTokens < LSP::RequestMessage + property params : LSP::SemanticTokensParams + + def execute(server) + uri = + URI.parse(params.text_document.uri) + + ast = + Workspace[uri.path.to_s][uri.path.to_s] + + # This is used later on to convert the line/column of each token + input = + ast.nodes.first.input + + tokenizer = SemanticTokenizer.new + tokenizer.tokenize(ast) + + data = + tokenizer.tokens.sort_by(&.from).compact_map do |token| + location = + Ast::Node.compute_location(input, token.from, token.to) + + type = + token.type.to_s.underscore + + if index = SemanticTokenizer::TOKEN_TYPES.index(type) + [ + location.start[0] - 1, + location.start[1], + token.to - token.from, + index, + 0, + ] + end + end + + result = [] of Array(Int32) + + data.each_with_index do |item, index| + current = + item.dup + + unless index.zero? + last = + data[index - 1] + + current[0] = + current[0] - last[0] + + current[1] = current[1] - last[1] if current[0] == 0 + end + + result << current + end + + { + data: result.flatten, + } + end + end + end +end diff --git a/src/ls/server.cr b/src/ls/server.cr index 98d3ba3c1..9166feae6 100644 --- a/src/ls/server.cr +++ b/src/ls/server.cr @@ -8,13 +8,14 @@ module Mint # Text document related methods method "textDocument/willSaveWaitUntil", WillSaveWaitUntil + method "textDocument/semanticTokens/full", SemanticTokens method "textDocument/foldingRange", FoldingRange method "textDocument/formatting", Formatting method "textDocument/completion", Completion method "textDocument/codeAction", CodeAction + method "textDocument/definition", Definition method "textDocument/didChange", DidChange method "textDocument/hover", Hover - method "textDocument/definition", Definition property params : LSP::InitializeParams? = nil diff --git a/src/lsp/protocol/semantic_tokens_client_capabilities.cr b/src/lsp/protocol/semantic_tokens_client_capabilities.cr new file mode 100644 index 000000000..09349a289 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_client_capabilities.cr @@ -0,0 +1,16 @@ +module LSP + struct SemanticTokensClientCapabilities + include JSON::Serializable + + # Whether definition supports dynamic registration. + @[JSON::Field(key: "dynamicRegistration")] + property dynamic_registration : Bool? + + # The token types that the client supports. + @[JSON::Field(key: "tokenTypes")] + property token_types : Array(String)? + + def initialize(@dynamic_registration = nil, @token_types = nil) + end + end +end diff --git a/src/lsp/protocol/semantic_tokens_legend.cr b/src/lsp/protocol/semantic_tokens_legend.cr new file mode 100644 index 000000000..55de94a39 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_legend.cr @@ -0,0 +1,16 @@ +module LSP + struct SemanticTokensLegend + include JSON::Serializable + + # The token types a server uses. + @[JSON::Field(key: "tokenTypes")] + property token_types : Array(String) + + # The token modifiers a server uses. + @[JSON::Field(key: "tokenModifiers")] + property token_modifiers : Array(String) + + def initialize(@token_types, @token_modifiers) + end + end +end diff --git a/src/lsp/protocol/semantic_tokens_options.cr b/src/lsp/protocol/semantic_tokens_options.cr new file mode 100644 index 000000000..49b2dd1b0 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_options.cr @@ -0,0 +1,18 @@ +module LSP + struct SemanticTokensOptions + include JSON::Serializable + + # The legend used by the server + property legend : SemanticTokensLegend + + # Server supports providing semantic tokens for a specific range + # of a document. + property? range : Bool + + # Server supports providing semantic tokens for a full document. + property? full : Bool + + def initialize(@legend, @range, @full) + end + end +end diff --git a/src/lsp/protocol/semantic_tokens_params.cr b/src/lsp/protocol/semantic_tokens_params.cr new file mode 100644 index 000000000..175b89d29 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_params.cr @@ -0,0 +1,9 @@ +module LSP + class SemanticTokensParams + include JSON::Serializable + + # The document in which the command was invoked. + @[JSON::Field(key: "textDocument")] + property text_document : TextDocumentIdentifier + end +end diff --git a/src/lsp/protocol/server_capabilities.cr b/src/lsp/protocol/server_capabilities.cr index b67cbef19..d3d1e18ca 100644 --- a/src/lsp/protocol/server_capabilities.cr +++ b/src/lsp/protocol/server_capabilities.cr @@ -96,6 +96,10 @@ module LSP @[JSON::Field(key: "executeCommandProvider")] property execute_command_provider : ExecuteCommandOptions + # The server provides semantic tokens support. + @[JSON::Field(key: "semanticTokensProvider")] + property semantic_tokens_provider : SemanticTokensOptions + # Workspace specific server capabilities property workspace : Workspace @@ -105,6 +109,7 @@ module LSP @document_formatting_provider, @document_highlight_provider, @workspace_symbol_provider, + @semantic_tokens_provider, @document_symbol_provider, @type_definition_provider, @execute_command_provider, diff --git a/src/lsp/protocol/text_document_client_capabilities.cr b/src/lsp/protocol/text_document_client_capabilities.cr index b2a8af71e..8aab5e82f 100644 --- a/src/lsp/protocol/text_document_client_capabilities.cr +++ b/src/lsp/protocol/text_document_client_capabilities.cr @@ -7,5 +7,9 @@ module LSP # Capabilities specific to the `textDocument/definition` request. property definition : DefinitionClientCapabilities? + + # Capabilities specific to the `textDocument/semanticTokens` request. + @[JSON::Field(key: "semanticTokens")] + property semantic_tokens : SemanticTokensClientCapabilities? end end diff --git a/src/parser.cr b/src/parser.cr index 0f6da5c6c..c2d9083a8 100644 --- a/src/parser.cr +++ b/src/parser.cr @@ -22,13 +22,16 @@ module Mint def start(&) start_position = position + node_size = ast.nodes.size begin node = yield position @position = start_position unless node + ast.nodes.delete_at(node_size...) unless node node rescue error : Error @position = start_position + ast.nodes.delete_at(node_size...) raise error end end @@ -180,6 +183,10 @@ module Mint def keyword(word) : Bool if keyword_ahead?(word) + if word.chars.all?(&.ascii_lowercase?) && !word.blank? && word != "or" + @ast.keywords << {position, position + word.size} + end + @position += word.size true else diff --git a/src/parsers/basic_expression.cr b/src/parsers/basic_expression.cr index 49a3e701b..051489ac5 100644 --- a/src/parsers/basic_expression.cr +++ b/src/parsers/basic_expression.cr @@ -3,6 +3,7 @@ module Mint # NOTE: The order of the parsing is important! def basic_expression : Ast::Expression? format_directive || + highlight_directive || documentation_directive || svg_directive || asset_directive || diff --git a/src/parsers/code_block.cr b/src/parsers/code_block.cr index 6c3717e27..1b943e522 100644 --- a/src/parsers/code_block.cr +++ b/src/parsers/code_block.cr @@ -1,5 +1,18 @@ module Mint class Parser + def code_block_naked : Ast::Block? + start do |start_position| + statements = + many { comment || statement } + + self << Ast::Block.new( + statements: statements, + from: start_position, + to: position, + input: data) if statements + end + end + def code_block : Ast::Block? start do |start_position| statements = diff --git a/src/parsers/comment.cr b/src/parsers/comment.cr index 44b4af94c..2ddad63b0 100644 --- a/src/parsers/comment.cr +++ b/src/parsers/comment.cr @@ -24,7 +24,7 @@ module Mint whitespace - Ast::Comment.new( + self << Ast::Comment.new( from: start_position, value: value, type: type, diff --git a/src/parsers/directives/highlight.cr b/src/parsers/directives/highlight.cr new file mode 100644 index 000000000..e273354a7 --- /dev/null +++ b/src/parsers/directives/highlight.cr @@ -0,0 +1,25 @@ +module Mint + class Parser + syntax_error FormatDirectiveExpectedOpeningBracket + syntax_error FormatDirectiveExpectedClosingBracket + syntax_error FormatDirectiveExpectedExpression + + def highlight_directive : Ast::Directives::Highlight? + start do |start_position| + next unless keyword "@highlight" + + content = + code_block( + opening_bracket: FormatDirectiveExpectedOpeningBracket, + closing_bracket: FormatDirectiveExpectedClosingBracket, + statement_error: FormatDirectiveExpectedExpression) + + self << Ast::Directives::Highlight.new( + from: start_position, + content: content, + to: position, + input: data) + end + end + end +end diff --git a/src/parsers/html_body.cr b/src/parsers/html_body.cr index b9a99d3d6..85183c3dd 100644 --- a/src/parsers/html_body.cr +++ b/src/parsers/html_body.cr @@ -44,6 +44,9 @@ module Mint tag end + closing_tag_position = + position + 2 + raise expected_closing_tag, position, { "opening_tag" => tag, } unless keyword "" @@ -60,7 +63,8 @@ module Mint {attributes, children, - comments} + comments, + closing_tag_position} end end end diff --git a/src/parsers/html_component.cr b/src/parsers/html_component.cr index 88d32eeb7..4147ac54e 100644 --- a/src/parsers/html_component.cr +++ b/src/parsers/html_component.cr @@ -21,13 +21,15 @@ module Mint variable! HtmlComponentExpectedReference end - attributes, children, comments = html_body( - expected_closing_bracket: HtmlComponentExpectedClosingBracket, - expected_closing_tag: HtmlComponentExpectedClosingTag, - with_dashes: false, - tag: component) + attributes, children, comments, closing_tag_position = + html_body( + expected_closing_bracket: HtmlComponentExpectedClosingBracket, + expected_closing_tag: HtmlComponentExpectedClosingTag, + with_dashes: false, + tag: component) node = self << Ast::HtmlComponent.new( + closing_tag_position: closing_tag_position, attributes: attributes, from: start_position, component: component, diff --git a/src/parsers/html_element.cr b/src/parsers/html_element.cr index b9a888091..5c2db7e21 100644 --- a/src/parsers/html_element.cr +++ b/src/parsers/html_element.cr @@ -30,13 +30,15 @@ module Mint variable! HtmlElementExpectedReference end - attributes, children, comments = html_body( - expected_closing_bracket: HtmlElementExpectedClosingBracket, - expected_closing_tag: HtmlElementExpectedClosingTag, - with_dashes: true, - tag: tag) + attributes, children, comments, closing_tag_position = + html_body( + expected_closing_bracket: HtmlElementExpectedClosingBracket, + expected_closing_tag: HtmlElementExpectedClosingTag, + with_dashes: true, + tag: tag) node = self << Ast::HtmlElement.new( + closing_tag_position: closing_tag_position, attributes: attributes, from: start_position, children: children, diff --git a/src/parsers/operation.cr b/src/parsers/operation.cr index 1ee471b8b..249db7cd0 100644 --- a/src/parsers/operation.cr +++ b/src/parsers/operation.cr @@ -30,9 +30,11 @@ module Mint def operator : String? start do whitespace + saved_position = position operator = OPERATORS.keys.find { |item| keyword item } next unless operator next unless whitespace? + ast.operators << {saved_position, saved_position + operator.size} whitespace operator end diff --git a/src/parsers/type_id.cr b/src/parsers/type_id.cr index 56f55916a..d8734533c 100644 --- a/src/parsers/type_id.cr +++ b/src/parsers/type_id.cr @@ -1,6 +1,6 @@ module Mint class Parser - def type_id!(error : SyntaxError.class) : Ast::TypeId + def type_id!(error : SyntaxError.class, *, track : Bool = true) : Ast::TypeId start do |start_position| value = gather do char(error, &.ascii_uppercase?) @@ -10,7 +10,7 @@ module Mint raise error unless value if char! '.' - other = type_id! error + other = type_id!(error, track: false) value += ".#{other.value}" end @@ -18,11 +18,13 @@ module Mint from: start_position, value: value, to: position, - input: data) + input: data).tap do |node| + self << node if track + end end end - def type_id : Ast::TypeId? + def type_id(*, track : Bool = true) : Ast::TypeId? start do |start_position| value = gather do return unless char.ascii_uppercase? @@ -36,7 +38,7 @@ module Mint if char == '.' other = start do step - next_part = type_id + next_part = type_id(track: false) next unless next_part next_part end @@ -51,13 +53,15 @@ module Mint from: start_position, value: value, to: position, - input: data) + input: data).tap do |node| + self << node if track + end end end - def type_id(error : SyntaxError.class) : Ast::TypeId? + def type_id(error : SyntaxError.class, *, track : Bool = true) : Ast::TypeId? return unless char.ascii_uppercase? - type_id! error + type_id! error, track: track end end end diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr new file mode 100644 index 000000000..c359bfcff --- /dev/null +++ b/src/semantic_tokenizer.cr @@ -0,0 +1,173 @@ +module Mint + class SemanticTokenizer + # This is a subset of the LSPs SemanticTokenTypes enum. + enum TokenType + TypeParameter + Type + + Namespace + Property + Keyword + Comment + + Variable + Operator + String + Number + Regexp + end + + TOKEN_TYPES = TokenType.names.map!(&.camelcase(lower: true)) + + # This represents which token types are used for which node. + TOKEN_MAP = { + Ast::TypeVariable => TokenType::TypeParameter, + Ast::Variable => TokenType::Variable, + Ast::Comment => TokenType::Comment, + Ast::StringLiteral => TokenType::String, + Ast::RegexpLiteral => TokenType::Regexp, + Ast::NumberLiteral => TokenType::Number, + Ast::TypeId => TokenType::Type, + } + + # Represents a semantic token using the positions of the token instead + # of line / column (for the LSP it is converted to line /column). + record Token, + type : TokenType, + from : Int32, + to : Int32 + + # We keep a cache of all tokenized nodes to avoid duplications + getter cache = Set(Ast::Node).new + + # This is where the resulting tokens are stored. + getter tokens = [] of Token + + def self.tokenize(ast : Ast) + tokenizer = self.new + tokenizer.tokenize(ast) + + parts = [] of String | Tuple(SemanticTokenizer::TokenType, String) + contents = ast.nodes.first.input.input + position = 0 + + tokenizer.tokens.sort_by(&.from).each do |token| + if token.from > position + parts << contents[position, token.from - position] + end + + parts << {token.type, contents[token.from, token.to - token.from]} + position = token.to + end + + if position < contents.size + parts << contents[position, contents.size] + end + + parts + end + + def self.highlight(path : String, html : Bool = false) + ast = + Parser.parse(path) + + parts = + tokenize(ast) + + parts.join do |item| + case item + in String + html ? HTML.escape(item) : item + in Tuple(SemanticTokenizer::TokenType, String) + if html + next "#{HTML.escape(item[1])}" + end + + case item[0] + in .type? + item[1].colorize(:yellow) + in .type_parameter? + item[1].colorize(:light_yellow) + in .variable? + item[1].colorize(:dark_gray) + in .namespace? + item[1].colorize(:light_blue) + in .keyword? + item[1].colorize(:magenta) + in .property? + item[1].colorize(:dark_gray).mode(:underline) + in .comment? + item[1].colorize(:light_gray) + in .string? + item[1].colorize(:green) + in .number? + item[1].colorize(:red) + in .regexp? + item[1].colorize(:light_red) + in .operator? + item[1].colorize(:light_magenta) + end.to_s + end + end + end + + def tokenize(ast : Ast) + # We add the operators and keywords directly from the AST + ast.operators.each { |(from, to)| add(from, to, :operator) } + ast.keywords.each { |(from, to)| add(from, to, :keyword) } + + tokenize(ast.nodes) + end + + def tokenize(nodes : Array(Ast::Node)) + nodes.each { |node| tokenize(node) } + end + + def tokenize(node : Ast::Node?) + if type = TOKEN_MAP[node.class]? + add(node, type) + end + end + + def tokenize(node : Ast::CssDefinition) + add(node.from, node.from + node.name.size, :property) + end + + def tokenize(node : Ast::ArrayAccess) + # TODO: The index should be parsed as a number literal when + # implemented remove this + case index = node.index + when Int64 + add(node.from + 1, node.from + 1 + index.to_s.size, :number) + end + end + + def tokenize(node : Ast::HtmlElement) + # The closing tag is not saved only the position to it. + node.closing_tag_position.try do |position| + add(position, position + node.tag.value.size, :namespace) + end + + add(node.tag, TokenType::Namespace) + end + + def tokenize(node : Ast::HtmlComponent) + node.closing_tag_position.try do |position| + add(position, position + node.component.value.size, :type) + end + end + + def add(from : Int32, to : Int32, type : TokenType) + tokens << Token.new( + type: type, + from: from, + to: to) + end + + def add(node : Ast::Node, type : TokenType) + return if cache.includes?(node) + add(node.from, node.to, type) + cache.add(node) + end + end +end diff --git a/src/type_checkers/directives/highlight.cr b/src/type_checkers/directives/highlight.cr new file mode 100644 index 000000000..9e47bf932 --- /dev/null +++ b/src/type_checkers/directives/highlight.cr @@ -0,0 +1,7 @@ +module Mint + class TypeChecker + def check(node : Ast::Directives::Highlight) : Checkable + Type.new("Tuple", [resolve(node.content), HTML] of Checkable) + end + end +end