feat(server)!: start to use publicodes-tree-sitter

.vscode/settings.json

@@ -3,6 +3,6 @@
 	"typescript.tsc.autoDetect": "off",
 	"typescript.preferences.quoteStyle": "single",
 	"editor.codeActionsOnSave": {
-		"source.fixAll.eslint": true
+		"source.fixAll.eslint": "explicit"
 	}
 }

server/package.json

@@ -1,24 +1,27 @@
 {
-	"name": "publicodes-language-server",
-	"description": "Language server for Publicodes",
-	"version": "0.1.0",
-	"author": "Emile Rolley <[email protected]>",
-	"license": "MIT",
-	"engines": {
-		"node": "*"
-	},
-	"repository": {
-		"type": "git",
-		"url": "https://github.com/Microsoft/vscode-extension-samples"
-	},
-	"dependencies": {
-		"fs": "^0.0.1-security",
-		"path": "^0.12.7",
-		"publicodes": "^1.0.0-beta.69",
-		"vscode-languageserver": "^8.1.0",
-		"vscode-languageserver-textdocument": "^1.0.8",
-		"yaml": "^2.2.1"
-	},
-	"scripts": {},
-	"devDependencies": {}
+  "name": "publicodes-language-server",
+  "description": "Language server for Publicodes",
+  "version": "0.1.0",
+  "author": "Emile Rolley <[email protected]>",
+  "license": "MIT",
+  "engines": {
+    "node": "*"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Microsoft/vscode-extension-samples"
+  },
+  "dependencies": {
+    "@publicodes/tools": "^1.0.7",
+    "fs": "^0.0.1-security",
+    "path": "^0.12.7",
+    "publicodes": "^1.2.0",
+    "tree-sitter": "^0.21.1",
+    "tree-sitter-publicodes": "file:../../tree-sitter-publicodes",
+    "vscode-languageserver": "^8.1.0",
+    "vscode-languageserver-textdocument": "^1.0.8",
+    "yaml": "^2.2.1"
+  },
+  "scripts": {},
+  "devDependencies": {}
 }

server/src/context.ts

@@ -4,6 +4,7 @@ import {
   Diagnostic,
   TextDocuments,
 } from "vscode-languageserver/node";
+import * as TSParser from "tree-sitter";
 
 export type GlobalConfig = {
   hasConfigurationCapability: boolean;
@@ -32,21 +33,48 @@ export type FilePath = string;
 
 export type DottedName = string;
 
+// TODO: use the publicodes types
 export type RawPublicodes = Record<DottedName, any>;
 
+export type FileInfos = {
+  // List of rules in the file extracted from the tree-sitter's CST
+  ruleDefs: RuleDef[];
+  // Raw publicodes rules extracted from the file (with resolved imports).
+  // It's used to be able to parse the rules with the publicodes engine.
+  rawRules: RawPublicodes;
+  // Tree-sitter CST of the file used to extract the rules.
+  // NOTE: It is stored to get more efficient parsing when the file is changed.
+  tsTree: TSParser.Tree;
+};
+
+export type RuleDef = {
+  kind: "rule" | "namespace" | "constant";
+  name: string;
+  pos: {
+    start: TSParser.Point;
+    end: TSParser.Point;
+  };
+};
+
 export type LSContext = {
   connection: Connection;
+  config: GlobalConfig;
+  globalSettings: DocumentSettings;
   rootFolderPath?: string;
   nodeModulesPaths?: string[];
   documents: TextDocuments<TextDocument>;
   documentSettings: Map<string, Thenable<DocumentSettings>>;
-  globalSettings: DocumentSettings;
-  config: GlobalConfig;
+  fileInfos: Map<FilePath, FileInfos>;
+  diagnostics: Diagnostic[];
+
+  // TODO: maybe to remove
   ruleToFileNameMap: Map<DottedName, FilePath>;
   fileNameToRulesMap: Map<FilePath, DottedName[]>;
-  URIToRevalidate: Set<FilePath>;
+
+  // TODO: to remove
   rawPublicodesRules: RawPublicodes;
   parsedRules: Record<string, any>;
   dirsToIgnore: string[];
   lastOpenedFile?: string;
+  URIToRevalidate: Set<FilePath>;
 };

server/src/initialized.ts

@@ -1,7 +1,7 @@
 import { DidChangeConfigurationNotification } from "vscode-languageserver/node";
 import { LSContext } from "./context";
 import { fileURLToPath } from "node:url";
-import { parseRawPublicodesRules } from "./publicodesRules";
+import { parseDir } from "./publicodesRules";
 import validate from "./validate";
 import { existsSync, statSync } from "fs";
 import { readdirSync } from "node:fs";
@@ -12,7 +12,7 @@ export default function intializedHandler(ctx: LSContext) {
       // Register for all configuration changes.
       ctx.connection.client.register(
         DidChangeConfigurationNotification.type,
-        undefined
+        undefined,
       );
     }
     if (ctx.config.hasWorkspaceFolderCapability) {
@@ -33,10 +33,10 @@ export default function intializedHandler(ctx: LSContext) {
             });
           }
           folders.forEach((folder) => {
-            ctx = parseRawPublicodesRules(ctx, folder.uri);
+            parseDir(ctx, folder.uri);
           });
           ctx.connection.console.log(
-            `Validating ${Object.keys(ctx.rawPublicodesRules).length} rules`
+            `Validating ${Object.keys(ctx.rawPublicodesRules).length} rules`,
           );
           validate(ctx);
         }

server/src/publicodesRules.ts

@@ -1,65 +1,29 @@
 import { readFileSync, readdirSync, statSync } from "fs";
 import { join } from "path";
-import { FilePath, LSContext } from "./context";
+import { FilePath, LSContext, RawPublicodes, RuleDef } from "./context";
 import parseYAML from "./parseYAML";
 import { fileURLToPath, pathToFileURL } from "node:url";
 
 import { resolveImports } from "./resolveImports";
 import { TextDocument } from "vscode-languageserver-textdocument";
 
-const PUBLICODES_FILE_EXTENSION = ".publicodes";
-
-export function parseRawPublicodesRulesFromDocument(
-  ctx: LSContext,
-  filePath: FilePath,
-  document?: TextDocument
-) {
-  const { rules, error } = parseYAML(
-    ctx,
-    document,
-    readFileSync(filePath).toString()
-  );
-  const resolvedRules = resolveImports(rules, { verbose: true, ctx });
-
-  const ruleNames = Object.keys(resolvedRules);
-
-  // Manage removed rules
-  if (ctx.fileNameToRulesMap.has(filePath)) {
-    ctx.fileNameToRulesMap
-      .get(filePath)
-      ?.filter((rule) => !ruleNames.includes(rule))
-      ?.forEach((rule) => {
-        ctx.ruleToFileNameMap.delete(rule);
-        delete resolvedRules[rule];
-        delete ctx.rawPublicodesRules[rule];
-      });
-  }
-
-  ctx.fileNameToRulesMap.set(filePath, ruleNames);
-  ruleNames.forEach((rule) => ctx.ruleToFileNameMap.set(rule, filePath));
-  ctx.rawPublicodesRules = {
-    ...ctx.rawPublicodesRules,
-    ...resolvedRules,
-  };
+import * as TSParser from "tree-sitter";
+import * as Publicodes from "tree-sitter-publicodes";
+import { readFile } from "fs/promises";
+import { Diagnostic } from "vscode-languageserver/node";
 
-  if (error) {
-    ctx.connection.sendDiagnostics({
-      uri: document !== undefined ? document.uri : pathToFileURL(filePath).href,
-      diagnostics: [error],
-    });
-  }
+const PUBLICODES_FILE_EXTENSION = ".publicodes";
 
-  return ctx;
-}
+// NOTE: could be moved to the LSContext
+const parser = new TSParser();
+parser.setLanguage(Publicodes);
 
-// Explore recursively all files in the workspace folder
-// and concat all yaml files into one string for parsing
-// Explore recursively all files in the workspace folder
-// and concat all yaml files into one string for parsing
-export function parseRawPublicodesRules(
-  ctx: LSContext,
-  uri: string
-): LSContext {
+/**
+ * Explore recursively all files in the workspace folder and concat all yaml files into one string for parsing
+ *
+ * PERF: file reading is synchronous, should be done in parallel
+ */
+export function parseDir(ctx: LSContext, uri: string) {
   const path = fileURLToPath(uri);
   const files = readdirSync(path);
   files?.forEach((file) => {
@@ -72,18 +36,103 @@ export function parseRawPublicodesRules(
       // TODO: should be all allowed extensions, temporary fix to test
       filePath.endsWith(".yaml")
     ) {
-      ctx = parseRawPublicodesRulesFromDocument(
+      parseDocument(
         ctx,
         filePath,
-        ctx.documents.get(pathToFileURL(filePath).href)
+        ctx.documents.get(pathToFileURL(filePath).href),
       );
     } else if (
       statSync(filePath)?.isDirectory() &&
       !ctx.dirsToIgnore.includes(file)
     ) {
-      ctx = parseRawPublicodesRules(ctx, `${uri}/${file}`);
+      parseDir(ctx, `${uri}/${file}`);
+    }
+  });
+}
+
+export function parseDocument(
+  ctx: LSContext,
+  filePath: FilePath,
+  document?: TextDocument,
+) {
+  const fileContent = readFileSync(filePath).toString();
+  const currentFileInfos = ctx.fileInfos.get(filePath);
+  const tsTree = parser.parse(fileContent, currentFileInfos?.tsTree);
+
+  const { rawRules, error } = parseRawRules(ctx, filePath, document);
+
+  ctx.fileInfos.set(filePath, {
+    ruleDefs: collectRuleDefs(tsTree),
+    rawRules: rawRules,
+    tsTree,
+  });
+
+  if (error) {
+    ctx.diagnostics.push(error);
+  }
+}
+
+/**
+ * Parse and resolve imports of a publicodes file
+ */
+function parseRawRules(
+  ctx: LSContext,
+  filePath: FilePath,
+  document?: TextDocument,
+): { rawRules: RawPublicodes; error?: Diagnostic | undefined } {
+  const { rules, error } = parseYAML(
+    ctx,
+    document,
+    readFileSync(filePath).toString(),
+  );
+
+  if (error) {
+    return { rawRules: {}, error: error };
+  }
+
+  const resolvedRules = resolveImports(rules, { verbose: true, ctx });
+
+  return { rawRules: resolvedRules };
+}
+
+/**
+ * Collects all rule definitions from the tree-sitter CST
+ *
+ * TODO: manage imbricated rule definitions
+ *
+ * @param tsTree - The tree-sitter CST of the file
+ * @return The list of rule definitions
+ */
+function collectRuleDefs(tsTree: TSParser.Tree): RuleDef[] {
+  const rules: RuleDef[] = [];
+
+  tsTree.rootNode.children.forEach((child) => {
+    switch (child.type) {
+      case "rule": {
+        const pos = { start: child.startPosition, end: child.endPosition };
+        const firstNamedChild = child.firstNamedChild;
+        if (!firstNamedChild) {
+          // TODO: manage error
+          return;
+        }
+
+        const ruleName = firstNamedChild.text;
+        const body = firstNamedChild.nextNamedSibling;
+
+        if (!body) {
+          return rules.push({ kind: "namespace", name: ruleName, pos });
+        }
+
+        return rules.push({
+          kind: body.type === "rule_body" ? "rule" : "constant",
+          name: ruleName,
+          pos,
+        });
+      }
+      case "comment":
+        return;
     }
   });
 
-  return ctx;
+  return rules;
 }

server/src/server.ts

@@ -34,6 +34,8 @@ let ctx: LSContext = {
     hasWorkspaceFolderCapability: false,
     hasDiagnosticRelatedInformationCapability: false,
   },
+  fileInfos: new Map(),
+  diagnostics: [],
   ruleToFileNameMap: new Map(),
   fileNameToRulesMap: new Map(),
   URIToRevalidate: new Set(),