Support documentation in definition files

CHANGELOG.md

@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+### Added
+
+- Added support for documentation in definitions files
+
+### Changed
+
+- Definitions files must now provide a name for the file in settings and command line: `--definitions:@roblox=path/to/globalTypes.d.luau`. Please update your LSP settings and command line arguments. Backwards compatibility has been temporarily preserved, with random names generated.
+
 ## [1.31.1] - 2024-07-07
 
 ### Fixed

editors/README.md

@@ -25,11 +25,11 @@ $ luau-lsp --help
 
 ## Configuring Definitions and Documentation
 
-You can add in built-in definitions by passing the `--definitions=PATH` argument.
+You can add in built-in definitions by passing the `--definitions:@name=PATH` argument. The `name` should be a unique reference to the definitions file.
 This can be done multiple times:
 
 ```sh
-$ luau-lsp lsp --definitions=/path/to/globalTypes.d.luau
+$ luau-lsp lsp --definitions:@roblox=/path/to/globalTypes.d.luau
 ```
 
 > NOTE: Definitions file syntax is unstable and undocumented. It may change at any time

editors/code/package.json

@@ -231,10 +231,10 @@
           "scope": "resource"
         },
         "luau-lsp.types.definitionFiles": {
-          "markdownDescription": "A list of paths to definition files to load in to the type checker. Note that definition file syntax is currently unstable and may change at any time",
-          "type": "array",
-          "default": [],
-          "items": {
+          "markdownDescription": "A mapping of package names to paths of definition files to load in to the type checker. Note that definition file syntax is currently unstable and may change at any time",
+          "type": "object",
+          "default": {},
+          "additionalProperties": {
             "type": "string"
           },
           "scope": "window"

editors/code/src/extension.ts

@@ -74,8 +74,17 @@ const startLanguageServer = async (context: vscode.ExtensionContext) => {
   const typesConfig = vscode.workspace.getConfiguration("luau-lsp.types");
 
   // Load extra type definitions
-  const definitionFiles = typesConfig.get<string[]>("definitionFiles");
+  let definitionFiles = typesConfig.get<
+    { [packageName: string]: string } | string[]
+  >("definitionFiles");
   if (definitionFiles) {
+    if (Array.isArray(definitionFiles)) {
+      // Convert to a map structure
+      definitionFiles = Object.fromEntries(
+        definitionFiles.map((path, index) => ["roblox" + index, path])
+      );
+    }
+
     for (const definitionPath of definitionFiles) {
       let uri;
       if (vscode.workspace.workspaceFolders) {

src/AnalyzeCli.cpp

@@ -149,7 +149,7 @@ int startAnalyze(const argparse::ArgumentParser& program)
     ReportFormat format = ReportFormat::Default;
     bool annotate = program.is_used("--annotate");
     auto sourcemapPath = program.present<std::filesystem::path>("--sourcemap");
-    auto definitionsPaths = program.get<std::vector<std::filesystem::path>>("--definitions");
+    auto definitionsPaths = processDefinitionsFilePaths(program);
     auto ignoreGlobPatterns = program.get<std::vector<std::string>>("--ignore");
     auto baseLuaurc = program.present<std::filesystem::path>("--base-luaurc");
     auto settingsPath = program.present<std::filesystem::path>("--settings");
@@ -310,7 +310,7 @@ int startAnalyze(const argparse::ArgumentParser& program)
     Luau::registerBuiltinGlobals(frontend, frontend.globals, /* typeCheckForAutocomplete = */ false);
     Luau::registerBuiltinGlobals(frontend, frontend.globalsForAutocomplete, /* typeCheckForAutocomplete = */ true);
 
-    for (auto& definitionsPath : definitionsPaths)
+    for (const auto& [packageName, definitionsPath] : definitionsPaths)
     {
         if (!std::filesystem::exists(definitionsPath))
         {
@@ -325,7 +325,8 @@ int startAnalyze(const argparse::ArgumentParser& program)
             return 1;
         }
 
-        auto loadResult = types::registerDefinitions(frontend, frontend.globals, *definitionsContents, /* typeCheckForAutocomplete = */ false);
+        auto loadResult =
+            types::registerDefinitions(frontend, frontend.globals, packageName, *definitionsContents, /* typeCheckForAutocomplete = */ false);
         if (!loadResult.success)
         {
             fprintf(stderr, "Failed to load definitions\n");

src/DocumentationParser.cpp

@@ -330,6 +330,26 @@ struct AttachCommentsVisitor : public Luau::AstVisitor
         return false;
     }
 
+    bool visit(Luau::AstStatDeclareClass* klass) override
+    {
+        if (klass->location.begin >= pos)
+            return false;
+        if (klass->location.begin > closestPreviousNode)
+            closestPreviousNode = klass->location.begin;
+
+        for (const auto& item : klass->props)
+        {
+            if (item.ty->location.begin >= pos)
+                continue;
+            closestPreviousNode = std::max(closestPreviousNode, item.ty->location.begin);
+            item.ty->visit(this);
+            if (item.ty->location.end <= pos)
+                closestPreviousNode = std::max(closestPreviousNode, item.ty->location.end);
+        }
+
+        return false;
+    }
+
     bool visit(Luau::AstStatBlock* block) override
     {
         // If the position is after the block, then it can be ignored
@@ -372,19 +392,30 @@ std::vector<Luau::Comment> getCommentLocations(const Luau::SourceModule* module,
 /// Performs transformations so that the comments are normalised to lines inside of it (i.e., trimming whitespace, removing comment start/end)
 std::vector<std::string> WorkspaceFolder::getComments(const Luau::ModuleName& moduleName, const Luau::Location& node)
 {
-    auto sourceModule = frontend.getSourceModule(moduleName);
-    if (!sourceModule)
-        return {};
+    Luau::SourceModule* sourceModule;
+    TextDocumentPtr textDocument(nullptr);
+
+    if (auto sm = definitionsSourceModules.find(moduleName); sm != definitionsSourceModules.end())
+    {
+        sourceModule = &sm->second.second;
+        textDocument = TextDocumentPtr(&sm->second.first);
+    }
+    else
+    {
+        sourceModule = frontend.getSourceModule(moduleName);
+        if (!sourceModule)
+            return {};
+
+        // Get relevant text document
+        textDocument = fileResolver.getOrCreateTextDocumentFromModuleName(moduleName);
+        if (!textDocument)
+            return {};
+    }
 
     auto commentLocations = getCommentLocations(sourceModule, node);
     if (commentLocations.empty())
         return {};
 
-    // Get relevant text document
-    auto textDocument = fileResolver.getOrCreateTextDocumentFromModuleName(moduleName);
-    if (!textDocument)
-        return {};
-
     std::vector<std::string> comments{};
     for (auto& comment : commentLocations)
     {
@@ -462,5 +493,9 @@ std::optional<std::string> WorkspaceFolder::getDocumentationForType(const Luau::
     {
         return printMoonwaveDocumentation(getComments(ttv->definitionModuleName, ttv->definitionLocation));
     }
+    else if (auto ctv = Luau::get<Luau::ClassType>(followedTy); ctv && !ctv->definitionModuleName.empty())
+    {
+        return printMoonwaveDocumentation(getComments(ctv->definitionModuleName, ctv->definitionLocation));
+    }
     return std::nullopt;
 }

src/LuauExt.cpp

@@ -51,11 +51,11 @@ std::optional<nlohmann::json> parseDefinitionsFileMetadata(const std::string& de
     return std::nullopt;
 }
 
-Luau::LoadDefinitionFileResult registerDefinitions(
-    Luau::Frontend& frontend, Luau::GlobalTypes& globals, const std::string& definitions, bool typeCheckForAutocomplete)
+Luau::LoadDefinitionFileResult registerDefinitions(Luau::Frontend& frontend, Luau::GlobalTypes& globals, const std::string& packageName,
+    const std::string& definitions, bool typeCheckForAutocomplete)
 {
-    // TODO: packageName shouldn't just be "@roblox"
-    return frontend.loadDefinitionFile(globals, globals.globalScope, definitions, "@roblox", /* captureComments = */ false, typeCheckForAutocomplete);
+    return frontend.loadDefinitionFile(
+        globals, globals.globalScope, definitions, packageName, /* captureComments = */ true, typeCheckForAutocomplete);
 }
 
 using NameOrExpr = std::variant<std::string, Luau::AstExpr*>;

src/Workspace.cpp

@@ -148,7 +148,7 @@ bool WorkspaceFolder::isDefinitionFile(const std::filesystem::path& path, const
     auto config = givenConfig ? *givenConfig : client->getConfiguration(rootUri);
     auto canonicalised = std::filesystem::weakly_canonical(path);
 
-    for (auto& file : config.types.definitionFiles)
+    for (const auto& [_, file] : client->definitionsFiles)
     {
         if (std::filesystem::weakly_canonical(file) == canonicalised)
         {
@@ -257,9 +257,9 @@ void WorkspaceFolder::initialize()
     if (client->definitionsFiles.empty())
         client->sendLogMessage(lsp::MessageType::Warning, "No definitions file provided by client");
 
-    for (const auto& definitionsFile : client->definitionsFiles)
+    for (const auto& [packageName, definitionsFile] : client->definitionsFiles)
     {
-        client->sendLogMessage(lsp::MessageType::Info, "Loading definitions file: " + definitionsFile.generic_string());
+        client->sendLogMessage(lsp::MessageType::Info, "Loading definitions file: " + packageName + " - " + definitionsFile.generic_string());
 
         auto definitionsContents = readFile(definitionsFile);
         if (!definitionsContents)
@@ -276,8 +276,10 @@ void WorkspaceFolder::initialize()
         client->sendTrace("workspace initialization: parsing definitions file metadata COMPLETED", json(definitionsFileMetadata).dump());
 
         client->sendTrace("workspace initialization: registering types definition");
-        auto result = types::registerDefinitions(frontend, frontend.globals, *definitionsContents, /* typeCheckForAutocomplete = */ false);
-        types::registerDefinitions(frontend, frontend.globalsForAutocomplete, *definitionsContents, /* typeCheckForAutocomplete = */ true);
+        auto result =
+            types::registerDefinitions(frontend, frontend.globals, packageName, *definitionsContents, /* typeCheckForAutocomplete = */ false);
+        types::registerDefinitions(
+            frontend, frontend.globalsForAutocomplete, packageName, *definitionsContents, /* typeCheckForAutocomplete = */ true);
         client->sendTrace("workspace initialization: registering types definition COMPLETED");
 
         auto uri = Uri::file(definitionsFile);
@@ -286,6 +288,8 @@ void WorkspaceFolder::initialize()
         {
             // Clear any set diagnostics
             client->publishDiagnostics({uri, std::nullopt, {}});
+            TextDocument textDocument(Uri::file(definitionsFile), "luau", 0, *definitionsContents);
+            definitionsSourceModules.insert_or_assign(packageName, std::make_pair(textDocument, result.sourceModule));
         }
         else
         {

src/include/Analyze/AnalyzeCli.hpp

@@ -1,6 +1,8 @@
 #pragma once
 #include <unordered_map>
+#include <filesystem>
 #include <string>
 #include "argparse/argparse.hpp"
 
+std::unordered_map<std::string, std::filesystem::path> processDefinitionsFilePaths(const argparse::ArgumentParser& program);
 int startAnalyze(const argparse::ArgumentParser& program);

src/include/LSP/Client.hpp

@@ -29,7 +29,7 @@ class Client : public BaseClient
     lsp::ClientCapabilities capabilities;
     lsp::TraceValue traceMode = lsp::TraceValue::Off;
     /// A registered definitions file passed by the client
-    std::vector<std::filesystem::path> definitionsFiles{};
+    std::unordered_map<std::string, std::filesystem::path> definitionsFiles{};
     /// A registered documentation file passed by the client
     std::vector<std::filesystem::path> documentationFiles{};
     /// Parsed documentation database

src/include/LSP/ClientConfiguration.hpp

@@ -32,9 +32,10 @@ struct ClientTypesConfiguration
     /// DEPRECATED: USE `platform.type` INSTEAD
     bool roblox = true;
     /// Any definition files to load globally
-    std::vector<std::filesystem::path> definitionFiles{};
+    // TODO: commented out due to backwards compatibility. uncomment if needed later once all settings have converted
+    // std::unordered_map<std::string, std::filesystem::path> definitionFiles{};
 };
-NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ClientTypesConfiguration, roblox, definitionFiles);
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ClientTypesConfiguration, roblox); // definitionFiles
 
 enum struct InlayHintsParameterNamesConfig
 {

src/include/LSP/DocumentationParser.hpp

@@ -13,6 +13,8 @@
 
 using json = nlohmann::json;
 
+const std::string kDocumentationBreaker = "\n----------\n";
+
 Luau::FunctionParameterDocumentation parseDocumentationParameter(const json& j);
 void parseDocumentation(
     const std::vector<std::filesystem::path>& documentationFiles, Luau::DocumentationDatabase& database, const std::shared_ptr<Client>& client);
@@ -25,4 +27,4 @@ std::optional<std::string> printDocumentation(const Luau::DocumentationDatabase&
 std::string printMoonwaveDocumentation(const std::vector<std::string>& comments);
 
 /// Get comments attached to a node (given the node's location)
-std::vector<Luau::Comment> getCommentLocations(const Luau::SourceModule* module, const Luau::Location& node);
+std::vector<Luau::Comment> getCommentLocations(const Luau::SourceModule* module, const Luau::Location& node);

src/include/LSP/LuauExt.hpp

@@ -18,8 +18,8 @@ bool isMetamethod(const Luau::Name& name);
 
 std::optional<nlohmann::json> parseDefinitionsFileMetadata(const std::string& definitions);
 
-Luau::LoadDefinitionFileResult registerDefinitions(
-    Luau::Frontend& frontend, Luau::GlobalTypes& globals, const std::string& definitions, bool typeCheckForAutocomplete = false);
+Luau::LoadDefinitionFileResult registerDefinitions(Luau::Frontend& frontend, Luau::GlobalTypes& globals, const std::string& packageName,
+    const std::string& definitions, bool typeCheckForAutocomplete = false);
 
 using NameOrExpr = std::variant<std::string, Luau::AstExpr*>;
 

src/include/LSP/Workspace.hpp

@@ -36,6 +36,9 @@ class WorkspaceFolder
     bool isConfigured = false;
     std::optional<nlohmann::json> definitionsFileMetadata;
 
+private:
+    std::unordered_map<std::string, std::pair<TextDocument, Luau::SourceModule>> definitionsSourceModules{};
+
 public:
     WorkspaceFolder(const std::shared_ptr<Client>& client, std::string name, const lsp::DocumentUri& uri, std::optional<Luau::Config> defaultConfig)
         : client(client)

src/main.cpp

@@ -13,6 +13,39 @@
 
 LUAU_FASTINT(LuauTarjanChildLimit)
 
+std::unordered_map<std::string, std::filesystem::path> processDefinitionsFilePaths(const argparse::ArgumentParser& program)
+{
+    std::unordered_map<std::string, std::filesystem::path> definitionsFiles{};
+    size_t backwardsCompatibilityNameSuffix = 0;
+    for (const auto& definition : program.get<std::vector<std::string>>("--definitions"))
+    {
+        std::string packageName = definition;
+        std::filesystem::path filePath = definition;
+
+        size_t eqIndex = definition.find('=');
+        if (eqIndex == std::string::npos)
+        {
+            // TODO: Remove Me - backwards compatibility
+            packageName = "@roblox";
+            if (backwardsCompatibilityNameSuffix > 0)
+                packageName += std::to_string(backwardsCompatibilityNameSuffix);
+            backwardsCompatibilityNameSuffix += 1;
+        }
+        else
+        {
+            packageName = definition.substr(0, eqIndex);
+            filePath = definition.substr(eqIndex + 1, definition.length());
+        }
+
+        if (!Luau::startsWith(packageName, "@"))
+            packageName = "@" + packageName;
+
+        definitionsFiles.emplace(packageName, filePath);
+    }
+
+    return definitionsFiles;
+}
+
 static void displayFlags()
 {
     printf("Available flags:\n");
@@ -45,7 +78,7 @@ int startLanguageServer(const argparse::ArgumentParser& program)
     _setmode(_fileno(stdout), _O_BINARY);
 #endif
 
-    auto definitionsFiles = program.get<std::vector<std::filesystem::path>>("--definitions");
+    auto definitionsFiles = processDefinitionsFilePaths(program);
     auto documentationFiles = program.get<std::vector<std::filesystem::path>>("--docs");
     std::optional<std::filesystem::path> baseLuaurc = program.present<std::filesystem::path>("--base-luaurc");
 
@@ -187,8 +220,7 @@ int main(int argc, char** argv)
         .metavar("PATH");
     analyze_command.add_argument("--definitions", "--defs")
         .help("A path to a Luau definitions file to load into the global namespace")
-        .action(file_path_parser)
-        .default_value<std::vector<std::filesystem::path>>({})
+        .default_value<std::vector<std::string>>({})
         .append()
         .metavar("PATH");
     analyze_command.add_argument("--ignore")

src/operations/Hover.cpp

@@ -343,19 +343,19 @@ std::optional<lsp::Hover> WorkspaceFolder::hover(const lsp::HoverParams& params)
     if (std::optional<std::string> docs;
         documentationSymbol && (docs = printDocumentation(client->documentation, *documentationSymbol)) && docs && !docs->empty())
     {
-        typeString += "\n----------\n";
+        typeString += kDocumentationBreaker;
         typeString += *docs;
     }
     else if (auto documentation = getDocumentationForType(*type); documentation && !documentation->empty())
     {
-        typeString += "\n----------\n";
+        typeString += kDocumentationBreaker;
         typeString += *documentation;
     }
     else if (documentationLocation)
     {
         if (auto text = printMoonwaveDocumentation(getComments(documentationLocation->moduleName, documentationLocation->location)); !text.empty())
         {
-            typeString += "\n----------\n";
+            typeString += kDocumentationBreaker;
             typeString += text;
         }
     }

src/platform/roblox/RobloxSourcemap.cpp

@@ -94,7 +94,7 @@ static Luau::TypeId getSourcemapType(const Luau::GlobalTypes& globals, Luau::Typ
 
             // Create the ClassType representing the instance
             std::string typeName = types::getTypeName(baseTypeId).value_or(node->name);
-            Luau::ClassType baseInstanceCtv{typeName, {}, baseTypeId, instanceMetaIdentity, {}, {}, "@roblox"};
+            Luau::ClassType baseInstanceCtv{typeName, {}, baseTypeId, instanceMetaIdentity, {}, {}, "@roblox", {}};
             auto typeId = arena.addType(std::move(baseInstanceCtv));
 
             // Attach Parent and Children info