Semantic Tokenizer (#615)

Co-authored-by: jansul <[email protected]>
Co-authored-by: Sijawusz Pur Rahnama <[email protected]>

spec/language_server/ls_spec.cr

@@ -5,7 +5,7 @@ def clean_json(workspace : Workspace, path : String)
 end
 
 Dir
-  .glob("./spec/language_server/{definition,hover}/**/*")
+  .glob("./spec/language_server/{definition,hover,semantic_tokens}/**/*")
   .select! { |file| File.file?(file) }
   .sort!
   .each do |file|

spec/language_server/semantic_tokens/record

@@ -0,0 +1,87 @@
+/* Some comment. */
+record Article {
+  id : Number,
+  description : String,
+  title : String
+}
+-----------------------------------------------------------------file test.mint
+{
+  "id": 0,
+  "method": "initialize",
+  "params": {
+    "capabilities": {
+      "textDocument": {
+        "semanticTokens": {
+          "dynamicRegistration": false,
+          "tokenTypes": ["property"]
+        }
+      }
+    }
+  }
+}
+-------------------------------------------------------------------------request
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "params": {
+    "textDocument": {
+      "uri": "file://#{root_path}/test.mint"
+    }
+  },
+  "method": "textDocument/semanticTokens/full"
+}
+-------------------------------------------------------------------------request
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "data": [
+      0,
+      0,
+      20,
+      5,
+      0,
+      1,
+      0,
+      6,
+      4,
+      0,
+      0,
+      7,
+      7,
+      1,
+      0,
+      1,
+      2,
+      2,
+      6,
+      0,
+      0,
+      5,
+      6,
+      1,
+      0,
+      1,
+      2,
+      11,
+      6,
+      0,
+      0,
+      14,
+      6,
+      1,
+      0,
+      1,
+      2,
+      5,
+      6,
+      0,
+      0,
+      8,
+      6,
+      1,
+      0
+    ]
+  },
+  "id": 1
+}
+------------------------------------------------------------------------response

src/all.cr

@@ -57,6 +57,8 @@ require "./documentation_generator/**"
 require "./documentation_generator"
 require "./documentation_server"
 
+require "./semantic_tokenizer"
+
 require "./test_runner/**"
 require "./test_runner"
 

src/ast.cr

@@ -31,9 +31,12 @@ module Mint
                        Js
 
     getter components, modules, records, stores, routes, providers
-    getter suites, enums, comments, nodes, unified_modules
+    getter suites, enums, comments, nodes, unified_modules, keywords
+    getter operators
 
-    def initialize(@records = [] of RecordDefinition,
+    def initialize(@operators = [] of Tuple(Int32, Int32),
+                   @keywords = [] of Tuple(Int32, Int32),
+                   @records = [] of RecordDefinition,
                    @unified_modules = [] of Module,
                    @components = [] of Component,
                    @providers = [] of Provider,

src/ast/directives/highlight.cr

@@ -0,0 +1,15 @@
+module Mint
+  class Ast
+    module Directives
+      class Highlight < Node
+        getter content
+
+        def initialize(@content : Block,
+                       @input : Data,
+                       @from : Int32,
+                       @to : Int32)
+        end
+      end
+    end
+  end
+end

src/ast/html_component.cr

@@ -1,13 +1,14 @@
 module Mint
   class Ast
     class HtmlComponent < Node
-      getter attributes, children, component, comments, ref
+      getter attributes, children, component, comments, ref, closing_tag_position
 
       def initialize(@attributes : Array(HtmlAttribute),
+                     @closing_tag_position : Int32?,
                      @comments : Array(Comment),
                      @children : Array(Node),
-                     @ref : Variable?,
                      @component : TypeId,
+                     @ref : Variable?,
                      @input : Data,
                      @from : Int32,
                      @to : Int32)

src/ast/html_element.cr

@@ -2,8 +2,10 @@ module Mint
   class Ast
     class HtmlElement < Node
       getter attributes, children, styles, tag, comments, ref
+      getter closing_tag_position
 
       def initialize(@attributes : Array(HtmlAttribute),
+                     @closing_tag_position : Int32?,
                      @comments : Array(Comment),
                      @styles : Array(HtmlStyle),
                      @children : Array(Node),

src/ast/node.cr

@@ -71,7 +71,7 @@ module Mint
         source.strip.includes?('\n')
       end
 
-      protected def compute_position(lines, needle) : Position
+      def self.compute_position(lines, needle) : Position
         line_start_pos, line = begin
           left, right = 0, lines.size - 1
           index = pos = 0
@@ -107,19 +107,23 @@ module Mint
         {line, column}
       end
 
-      getter location : Location do
+      def self.compute_location(input : Data, from, to)
         # TODO: avoid creating this array for every (initial) call to `Node#location`
         lines = [0]
-        @input.input.each_char_with_index do |ch, i|
+        input.input.each_char_with_index do |ch, i|
           lines << i + 1 if ch == '\n'
         end
 
         Location.new(
-          filename: @input.file,
+          filename: input.file,
           start: compute_position(lines, from),
           end: compute_position(lines, to),
         )
       end
+
+      getter location : Location do
+        Node.compute_location(input, from, to)
+      end
     end
   end
 end

src/cli.cr

@@ -15,6 +15,7 @@ module Mint
     define_help description: "Mint"
 
     register_sub_command "sandbox-server", type: SandboxServer
+    register_sub_command highlight, type: Highlight
     register_sub_command install, type: Install
     register_sub_command compile, type: Compile
     register_sub_command version, type: Version

src/commands/highlight.cr

@@ -0,0 +1,20 @@
+module Mint
+  class Cli < Admiral::Command
+    class Highlight < Admiral::Command
+      include Command
+
+      define_help description: "Returns the syntax highlighted version of the given file"
+
+      define_argument path, description: "The path to the file"
+
+      define_flag html : Bool,
+        description: "If specified, print the highlighted code as HTML",
+        default: false
+
+      def run
+        return unless path = arguments.path
+        puts SemanticTokenizer.highlight(path, flags.html)
+      end
+    end
+  end
+end

src/compilers/directives/highlight.cr

@@ -0,0 +1,36 @@
+module Mint
+  class Compiler
+    def _compile(node : Ast::Directives::Highlight) : String
+      content =
+        compile node.content
+
+      formatted =
+        Formatter.new.format(node.content, Formatter::BlockFormat::Naked)
+
+      parser = Parser.new(formatted, "source.mint")
+      parser.code_block_naked
+
+      parts =
+        SemanticTokenizer.tokenize(parser.ast)
+
+      mapped =
+        parts.map do |item|
+          case item
+          in String
+            "`#{skip { escape_for_javascript(item) }}`"
+          in Tuple(SemanticTokenizer::TokenType, String)
+            "_h('span', { className: '#{item[0].to_s.underscore}' }, [`#{skip { escape_for_javascript(item[1]) }}`])"
+          end
+        end
+
+      "[#{content}, _h(React.Fragment, {}, [#{mapped.join(",\n")}])]"
+    end
+
+    def escape_for_javascript(value : String)
+      value
+        .gsub('\\', "\\\\")
+        .gsub('`', "\\`")
+        .gsub("${", "\\${")
+    end
+  end
+end

src/formatters/directives/highlight.cr

@@ -0,0 +1,7 @@
+module Mint
+  class Formatter
+    def format(node : Ast::Directives::Highlight)
+      "@highlight #{format(node.content)}"
+    end
+  end
+end

src/ls/definition/type_id.cr

@@ -0,0 +1,24 @@
+module Mint
+  module LS
+    class Definition < LSP::RequestMessage
+      def definition(node : Ast::TypeId, server : Server, workspace : Workspace, stack : Array(Ast::Node))
+        found =
+          workspace.ast.enums.find(&.name.value.==(node.value)) ||
+            workspace.ast.records.find(&.name.value.==(node.value)) ||
+            workspace.ast.stores.find(&.name.value.==(node.value)) ||
+            find_component(workspace, node.value)
+
+        if found.nil? && (next_node = stack[1])
+          return definition(next_node, server, workspace, stack)
+        end
+
+        return if Core.ast.nodes.includes?(found)
+
+        case found
+        when Ast::Store, Ast::Enum, Ast::Component, Ast::RecordDefinition
+          location_link server, node, found.name, found
+        end
+      end
+    end
+  end
+end

src/ls/initialize.cr

@@ -42,10 +42,20 @@ module Mint
               change_notifications: false,
               supported: false))
 
+        semantic_tokens_provider =
+          LSP::SemanticTokensOptions.new(
+            range: false,
+            full: true,
+            legend: LSP::SemanticTokensLegend.new(
+              token_types: SemanticTokenizer::TOKEN_TYPES,
+              token_modifiers: [] of String,
+            ))
+
         capabilities =
           LSP::ServerCapabilities.new(
             document_on_type_formatting_provider: document_on_type_formatting_provider,
             execute_command_provider: execute_command_provider,
+            semantic_tokens_provider: semantic_tokens_provider,
             signature_help_provider: signature_help_provider,
             document_link_provider: document_link_provider,
             completion_provider: completion_provider,

src/ls/semantic_tokens.cr

@@ -0,0 +1,65 @@
+module Mint
+  module LS
+    # This is the class that handles the "textDocument/semanticTokens/full" request.
+    class SemanticTokens < LSP::RequestMessage
+      property params : LSP::SemanticTokensParams
+
+      def execute(server)
+        uri =
+          URI.parse(params.text_document.uri)
+
+        ast =
+          Workspace[uri.path.to_s][uri.path.to_s]
+
+        # This is used later on to convert the line/column of each token
+        input =
+          ast.nodes.first.input
+
+        tokenizer = SemanticTokenizer.new
+        tokenizer.tokenize(ast)
+
+        data =
+          tokenizer.tokens.sort_by(&.from).compact_map do |token|
+            location =
+              Ast::Node.compute_location(input, token.from, token.to)
+
+            type =
+              token.type.to_s.underscore
+
+            if index = SemanticTokenizer::TOKEN_TYPES.index(type)
+              [
+                location.start[0] - 1,
+                location.start[1],
+                token.to - token.from,
+                index,
+                0,
+              ]
+            end
+          end
+
+        result = [] of Array(Int32)
+
+        data.each_with_index do |item, index|
+          current =
+            item.dup
+
+          unless index.zero?
+            last =
+              data[index - 1]
+
+            current[0] =
+              current[0] - last[0]
+
+            current[1] = current[1] - last[1] if current[0] == 0
+          end
+
+          result << current
+        end
+
+        {
+          data: result.flatten,
+        }
+      end
+    end
+  end
+end