From 6dee330acc9bb786cf5e56c6535213c3005a6d95 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sat, 16 Feb 2019 15:24:49 +0000 Subject: [PATCH 1/7] Add workspace symbol support --- pyls/hookspecs.py | 5 + pyls/lsp.py | 15 +++ pyls/plugins/ctags.py | 248 +++++++++++++++++++++++++++++++++++++ pyls/python_ls.py | 13 +- setup.py | 1 + test/plugins/test_ctags.py | 29 +++++ vscode-client/package.json | 43 +++++++ 7 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 pyls/plugins/ctags.py create mode 100644 test/plugins/test_ctags.py diff --git a/pyls/hookspecs.py b/pyls/hookspecs.py index d094dcb5..f64c8c69 100644 --- a/pyls/hookspecs.py +++ b/pyls/hookspecs.py @@ -110,3 +110,8 @@ def pyls_settings(config): @hookspec(firstresult=True) def pyls_signature_help(config, workspace, document, position): pass + + +@hookspec +def pyls_workspace_symbols(config, workspace, query): + pass diff --git a/pyls/lsp.py b/pyls/lsp.py index 36a8d842..f3d1d9ec 100644 --- a/pyls/lsp.py +++ b/pyls/lsp.py @@ -24,6 +24,13 @@ class CompletionItemKind(object): Color = 16 File = 17 Reference = 18 + Folder = 19 + EnumMember = 20 + Constant = 21 + Struct = 22 + Event = 23 + Operator = 24 + TypeParameter = 25 class DocumentHighlightKind(object): @@ -70,6 +77,14 @@ class SymbolKind(object): Number = 16 Boolean = 17 Array = 18 + Object = 19 + Key = 20 + Null = 21 + EnumMember = 22 + Struct = 23 + Event = 24 + Operator = 25 + TypeParameter = 26 class TextDocumentSyncKind(object): diff --git a/pyls/plugins/ctags.py b/pyls/plugins/ctags.py new file mode 100644 index 00000000..2c413257 --- /dev/null +++ b/pyls/plugins/ctags.py @@ -0,0 +1,248 @@ +# Copyright 2017 Palantir Technologies, Inc. +import io +import logging +import os +import re +import subprocess + + +from pyls import hookimpl, uris +from pyls.lsp import SymbolKind + +log = logging.getLogger(__name__) + +DEFAULT_TAG_FILE = "${workspaceFolder}/.vscode/tags" +DEFAULT_CTAGS_EXE = "ctags" + +TAG_RE = re.compile(( + r'(?P\w+)\t' + r'(?P.*)\t' + r'/\^(?P.*)\$/;"\t' + r'kind:(?P\w+)\t' + r'line:(?P\d+)$' +)) + +CTAG_OPTIONS = [ + "--tag-relative=yes", + "--exclude=.git", + "--exclude=env", + "--exclude=log", + "--exclude=tmp", + "--exclude=doc", + "--exclude=deps", + "--exclude=node_modules", + "--exclude=.vscode", + "--exclude=public/assets", + "--exclude=*.git*", + "--exclude=*.pyc", + "--exclude=*.pyo", + "--exclude=.DS_Store", + "--exclude=**/*.jar", + "--exclude=**/*.class", + "--exclude=**/.idea/", + "--exclude=build", + "--exclude=Builds", + "--exclude=doc", + "--fields=Knz", + "--extra=+f", +] + +CTAG_SYMBOL_MAPPING = { + "array": SymbolKind.Array, + "boolean": SymbolKind.Boolean, + "class": SymbolKind.Class, + "classes": SymbolKind.Class, + "constant": SymbolKind.Constant, + "constants": SymbolKind.Constant, + "constructor": SymbolKind.Constructor, + "enum": SymbolKind.Enum, + "enums": SymbolKind.Enum, + "enumeration": SymbolKind.Enum, + "enumerations": SymbolKind.Enum, + "field": SymbolKind.Field, + "fields": SymbolKind.Field, + "file": SymbolKind.File, + "files": SymbolKind.File, + "function": SymbolKind.Function, + "functions": SymbolKind.Function, + "member": SymbolKind.Function, + "interface": SymbolKind.Interface, + "interfaces": SymbolKind.Interface, + "key": SymbolKind.Key, + "keys": SymbolKind.Key, + "method": SymbolKind.Method, + "methods": SymbolKind.Method, + "module": SymbolKind.Module, + "modules": SymbolKind.Module, + "namespace": SymbolKind.Namespace, + "namespaces": SymbolKind.Namespace, + "number": SymbolKind.Number, + "numbers": SymbolKind.Number, + "null": SymbolKind.Null, + "object": SymbolKind.Object, + "package": SymbolKind.Package, + "packages": SymbolKind.Package, + "property": SymbolKind.Property, + "properties": SymbolKind.Property, + "objects": SymbolKind.Object, + "string": SymbolKind.String, + "variable": SymbolKind.Variable, + "variables": SymbolKind.Variable, + "projects": SymbolKind.Package, + "defines": SymbolKind.Module, + "labels": SymbolKind.Interface, + "macros": SymbolKind.Function, + "types (structs and records)": SymbolKind.Class, + "subroutine": SymbolKind.Method, + "subroutines": SymbolKind.Method, + "types": SymbolKind.Class, + "programs": SymbolKind.Class, + "Object\'s method": SymbolKind.Method, + "Module or functor": SymbolKind.Module, + "Global variable": SymbolKind.Variable, + "Type name": SymbolKind.Class, + "A function": SymbolKind.Function, + "A constructor": SymbolKind.Constructor, + "An exception": SymbolKind.Class, + "A \'structure\' field": SymbolKind.Field, + "procedure": SymbolKind.Function, + "procedures": SymbolKind.Function, + "constant definitions": SymbolKind.Constant, + "javascript functions": SymbolKind.Function, + "singleton methods": SymbolKind.Method, +} + + +class CtagMode(object): + NONE = "none" + APPEND = "append" + REBUILD = "rebuild" + + +DEFAULT_ON_START_MODE = CtagMode.REBUILD +DEFAULT_ON_SAVE_MODE = CtagMode.APPEND + + +class CtagsPlugin(object): + + def __init__(self): + self._started = False + self._workspace = None + + @hookimpl + def pyls_document_did_open(self, config, workspace): + """Since initial settings are sent after initialization, we use didOpen as the hook instead.""" + if self._started: + return + self._started = True + self._workspace = workspace + + settings = config.plugin_settings('ctags') + ctags_exe = _ctags_exe(settings) + + for tag_file in settings.get('tagFiles', []): + mode = tag_file.get('onStart', CtagMode.DEFAULT_ON_START_MODE) + + if mode == CtagMode.NONE: + log.debug("Skipping tag file with onStart mode NONE: %s", tag_file) + continue + + tag_file_path = self._format_path(tag_file['filePath']) + tags_dir = self._format_path(tag_file['directory']) + + execute(ctags_exe, tag_file_path, tags_dir, mode == CtagMode.APPEND) + + @hookimpl + def pyls_document_did_save(self, config, document): + settings = config.plugin_settings('ctags') + ctags_exe = _ctags_exe(settings) + + for tag_file in settings.get('tagFiles', []): + mode = tag_file.get('onSave', CtagMode.DEFAULT_ON_SAVE_MODE) + + if mode == CtagMode.NONE: + log.debug("Skipping tag file with onSave mode NONE: %s", tag_file) + continue + + tag_file_path = self._format_path(tag_file['filePath']) + tags_dir = self._format_path(tag_file['directory']) + + if not os.path.normpath(document.path).startswith(os.path.normpath(tags_dir)): + log.debug("Skipping onSave tag generation since %s is not in %s", tag_file_path, tags_dir) + continue + + execute(ctags_exe, tag_file_path, tags_dir, mode == CtagMode.APPEND) + + @hookimpl + def pyls_workspace_symbols(self, config, query): + settings = config.plugin_settings('ctags') + + symbols = [] + for tag_file in settings.get('tagFiles', []): + symbols.extend(parse_tags(self._format_path(tag_file['filePath']), query)) + + return symbols + + def _format_path(self, path): + return path.format(**{"workspaceRoot": self._workspace.root_path}) + + +def _ctags_exe(settings): + # TODO(gatesn): verify ctags is installed and right version + return settings.get('ctagsPath', DEFAULT_CTAGS_EXE) + + +def execute(ctags_exe, tag_file, directory, append=False): + """Run ctags against the given directory.""" + # Ensure the directory exists + tag_file_dir = os.path.dirname(tag_file) + if not os.path.exists(tag_file_dir): + os.makedirs(tag_file_dir) + + cmd = [ctags_exe, '-f', tag_file, '--languages=Python', '-R'] + CTAG_OPTIONS + if append: + cmd.append('--append') + cmd.append(directory) + + log.info("Executing exuberant ctags: %s", cmd) + log.info("ctags: %s", subprocess.check_output(cmd)) + + +def parse_tags(tag_file, query): + if not os.path.exists(tag_file): + return [] + + with io.open(tag_file, 'rb') as f: + for line in f: + tag = parse_tag(line.decode('utf-8', errors='ignore'), query) + if tag: + yield tag + + +def parse_tag(line, query): + match = TAG_RE.match(line) + if not match: + return None + + name = match.group('name') + + # TODO(gatesn): Support a fuzzy match, but for now do a naive substring match + if query.lower() not in name.lower(): + return None + + line = int(match.group('line')) - 1 + + return { + 'name': name, + 'kind': CTAG_SYMBOL_MAPPING.get(match.group('type'), SymbolKind.Null), + 'location': { + 'uri': uris.from_fs_path(match.group('file')), + 'range': { + 'start': {'line': line, 'character': 0}, + 'end': {'line': line, 'character': 0} + } + } + } + + +INSTANCE = CtagsPlugin() diff --git a/pyls/python_ls.py b/pyls/python_ls.py index d4275f5d..6c5d81b4 100644 --- a/pyls/python_ls.py +++ b/pyls/python_ls.py @@ -144,6 +144,7 @@ def capabilities(self): 'triggerCharacters': ['(', ','] }, 'textDocumentSync': lsp.TextDocumentSyncKind.INCREMENTAL, + 'workspaceSymbolProvider': True, 'experimental': merge(self._hook('pyls_experimental_capabilities')) } log.info('Server capabilities: %s', server_capabilities) @@ -230,6 +231,12 @@ def rename(self, doc_uri, position, new_name): def signature_help(self, doc_uri, position): return self._hook('pyls_signature_help', doc_uri, position=position) + def workspace_symbols(self, query): + if len(query) < 3: + # Avoid searching for symbols with no query + return None + return flatten(self._hook('pyls_workspace_symbols', query=query)) + def m_text_document__did_close(self, textDocument=None, **_kwargs): self.workspace.rm_document(textDocument['uri']) @@ -248,6 +255,7 @@ def m_text_document__did_change(self, contentChanges=None, textDocument=None, ** self.lint(textDocument['uri']) def m_text_document__did_save(self, textDocument=None, **_kwargs): + self._hook('pyls_document_did_save', textDocument['uri']) self.lint(textDocument['uri']) def m_text_document__code_action(self, textDocument=None, range=None, context=None, **_kwargs): @@ -299,9 +307,12 @@ def m_workspace__did_change_watched_files(self, **_kwargs): for doc_uri in self.workspace.documents: self.lint(doc_uri) - def m_workspace__execute_command(self, command=None, arguments=None): + def m_workspace__execute_command(self, command=None, arguments=None, **_kwargs): return self.execute_command(command, arguments) + def m_workspace__symbol(self, query=None, **_kwargs): + return self.workspace_symbols(query) + def flatten(list_of_lists): return [item for lst in list_of_lists for item in lst] diff --git a/setup.py b/setup.py index 441ded14..37f541e2 100755 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ ], 'pyls': [ 'autopep8 = pyls.plugins.autopep8_format', + 'ctags = pyls.plugins.ctags:INSTANCE', 'jedi_completion = pyls.plugins.jedi_completion', 'jedi_definition = pyls.plugins.definition', 'jedi_hover = pyls.plugins.hover', diff --git a/test/plugins/test_ctags.py b/test/plugins/test_ctags.py new file mode 100644 index 00000000..c4b661b1 --- /dev/null +++ b/test/plugins/test_ctags.py @@ -0,0 +1,29 @@ +# Copyright 2017 Palantir Technologies, Inc. +import os +import tempfile + +import pytest + +import pyls +from pyls import lsp +from pyls.plugins import ctags + + +@pytest.fixture(scope='session') +def pyls_ctags(): + """Fixture for generating ctags for the Python Langyage Server""" + _fd, tag_file = tempfile.mkstemp() + try: + ctags.execute("ctags", tag_file, os.path.dirname(pyls.__file__)) + yield tag_file + finally: + os.unlink(tag_file) + + +def test_parse_tags(pyls_ctags): + # Search for CtagsPlugin with the query 'tagsplug' + plugin_symbol = next(ctags.parse_tags(pyls_ctags, "tagsplug")) + + assert plugin_symbol['name'] == 'CtagsPlugin' + assert plugin_symbol['kind'] == lsp.SymbolKind.Class + assert plugin_symbol['location']['uri'].endswith('pyls/plugins/ctags.py') diff --git a/vscode-client/package.json b/vscode-client/package.json index 2f43ed1e..a69aba36 100644 --- a/vscode-client/package.json +++ b/vscode-client/package.json @@ -30,6 +30,49 @@ }, "uniqueItems": true }, + "pyls.plugins.ctags.enabled": { + "type": "boolean", + "default": true, + "description": "Enable or disable the plugin." + }, + "pyls.plugins.ctags.ctagsPath": { + "type": "string", + "default": "ctags", + "description": "Path to the ctags executable." + }, + "pyls.plugins.ctags.tagFiles": { + "type": "array", + "description": "Set of Exuberant CTags files for workspace symbols.", + "default": [{ + "filePath": "{workspaceRoot}/.vscode/tags", + "directory": "{workspaceRoot}" + }], + "items": { + "type": "object", + "properties": { + "filePath": { + "type": "string", + "description": "Path to the Exuberant Ctags tag file, can template {workspaceRoot} into the path." + }, + "directory": { + "type": "string", + "description": "Directory to index with ctags, can template {workspaceRoot} into the path." + }, + "onStart": { + "type": "string", + "enum": ["none", "append", "rebuild"], + "default": "rebuild", + "description": "Whether to rebuild or append the tags file on language server startup" + }, + "onSave": { + "type": "string", + "enum": ["none", "append", "rebuild"], + "default": "append", + "description": "Whether to rebuild or append the tags file when a file within the rootPath is saved." + } + } + } + }, "pyls.plugins.jedi_completion.enabled": { "type": "boolean", "default": true, From 60b50d1468fac227832d9d5474c0af3757ac1271 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sun, 17 Feb 2019 11:06:45 +0000 Subject: [PATCH 2/7] Add workspace symbol support --- .circleci/config.yml | 4 ++++ README.rst | 2 +- appveyor.yml | 7 +++++-- pyls/plugins/ctags.py | 6 +++--- test/plugins/test_ctags.py | 1 + 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da85eda1..9c0af1e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,8 @@ jobs: - image: "python:2.7-stretch" steps: - checkout + - run: apt-get update + - run: apt-get install exuberant-ctags - run: pip install -e .[all] .[test] - run: py.test test/ - run: pylint pyls test @@ -17,6 +19,8 @@ jobs: - image: "python:3.4-stretch" steps: - checkout + - run: apt-get update + - run: apt-get install exuberant-ctags - run: pip install -e .[all] .[test] - run: py.test test/ diff --git a/README.rst b/README.rst index 26ad2d2a..36ea4dcf 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ All optional providers can be installed using: ``pip install 'python-language-server[all]'`` -If you get an error similar to ``'install_requires' must be a string or list of strings`` then please upgrade setuptools before trying again. +If you get an error similar to ``'install_requires' must be a string or list of strings`` then please upgrade setuptools before trying again. ``pip install -U setuptools`` diff --git a/appveyor.yml b/appveyor.yml index 530e800d..1c8dfa51 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,8 +15,11 @@ init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" install: - - "%PYTHON%/python.exe -m pip install --upgrade pip setuptools" - - "%PYTHON%/python.exe -m pip install .[all] .[test]" + - 'appveyor DownloadFile "https://github.com/universal-ctags/ctags-win32/releases/download/2018-03-13/5010e849/ctags-2018-03-13_5010e849-x64.zip" -FileName ctags.zip' + - '7z e ctags.zip -oC:\Users\appveyor\bin ctags.exe' + - 'set PATH=%PATH%;C:\Users\appveyor\bin' + - '%PYTHON%/python.exe -m pip install --upgrade pip setuptools' + - '%PYTHON%/python.exe -m pip install .[all] .[test]' test_script: - "%PYTHON%/Scripts/pytest.exe test/" diff --git a/pyls/plugins/ctags.py b/pyls/plugins/ctags.py index 2c413257..6e2fa995 100644 --- a/pyls/plugins/ctags.py +++ b/pyls/plugins/ctags.py @@ -141,7 +141,7 @@ def pyls_document_did_open(self, config, workspace): ctags_exe = _ctags_exe(settings) for tag_file in settings.get('tagFiles', []): - mode = tag_file.get('onStart', CtagMode.DEFAULT_ON_START_MODE) + mode = tag_file.get('onStart', DEFAULT_ON_START_MODE) if mode == CtagMode.NONE: log.debug("Skipping tag file with onStart mode NONE: %s", tag_file) @@ -158,7 +158,7 @@ def pyls_document_did_save(self, config, document): ctags_exe = _ctags_exe(settings) for tag_file in settings.get('tagFiles', []): - mode = tag_file.get('onSave', CtagMode.DEFAULT_ON_SAVE_MODE) + mode = tag_file.get('onSave', DEFAULT_ON_SAVE_MODE) if mode == CtagMode.NONE: log.debug("Skipping tag file with onSave mode NONE: %s", tag_file) @@ -210,7 +210,7 @@ def execute(ctags_exe, tag_file, directory, append=False): def parse_tags(tag_file, query): if not os.path.exists(tag_file): - return [] + return with io.open(tag_file, 'rb') as f: for line in f: diff --git a/test/plugins/test_ctags.py b/test/plugins/test_ctags.py index c4b661b1..3cec914a 100644 --- a/test/plugins/test_ctags.py +++ b/test/plugins/test_ctags.py @@ -1,4 +1,5 @@ # Copyright 2017 Palantir Technologies, Inc. +# pylint: disable=redefined-outer-name import os import tempfile From d9f3ed48f57a1740f4788a6a76039c40dd9ec16d Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sun, 17 Feb 2019 11:15:59 +0000 Subject: [PATCH 3/7] Add workspace symbol support --- test/plugins/test_ctags.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/plugins/test_ctags.py b/test/plugins/test_ctags.py index 3cec914a..ec55f00f 100644 --- a/test/plugins/test_ctags.py +++ b/test/plugins/test_ctags.py @@ -13,7 +13,9 @@ @pytest.fixture(scope='session') def pyls_ctags(): """Fixture for generating ctags for the Python Langyage Server""" - _fd, tag_file = tempfile.mkstemp() + fd, tag_file = tempfile.mkstemp() + os.close(fd) # Close our handle to the file, we just want the path + try: ctags.execute("ctags", tag_file, os.path.dirname(pyls.__file__)) yield tag_file From c2267ff596d867388601421a537dcacac111cde1 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sun, 17 Feb 2019 11:28:49 +0000 Subject: [PATCH 4/7] Test windows ctags --- pyls/plugins/ctags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyls/plugins/ctags.py b/pyls/plugins/ctags.py index 6e2fa995..06539310 100644 --- a/pyls/plugins/ctags.py +++ b/pyls/plugins/ctags.py @@ -213,6 +213,7 @@ def parse_tags(tag_file, query): return with io.open(tag_file, 'rb') as f: + raise Exception(f.read()) for line in f: tag = parse_tag(line.decode('utf-8', errors='ignore'), query) if tag: From dafd5075bdfe96133a92fa80589afc7503559778 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sun, 17 Feb 2019 11:32:29 +0000 Subject: [PATCH 5/7] Test windows ctags --- pyls/plugins/ctags.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyls/plugins/ctags.py b/pyls/plugins/ctags.py index 06539310..97165b32 100644 --- a/pyls/plugins/ctags.py +++ b/pyls/plugins/ctags.py @@ -213,7 +213,6 @@ def parse_tags(tag_file, query): return with io.open(tag_file, 'rb') as f: - raise Exception(f.read()) for line in f: tag = parse_tag(line.decode('utf-8', errors='ignore'), query) if tag: @@ -222,6 +221,8 @@ def parse_tags(tag_file, query): def parse_tag(line, query): match = TAG_RE.match(line) + log.info("Got match %s from line: %s", match, line) + if not match: return None From 8a3f9604cd51f0ea50dbcd2766fc862cad6c0d5e Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sun, 17 Feb 2019 11:44:24 +0000 Subject: [PATCH 6/7] Test windows ctags --- pyls/plugins/ctags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyls/plugins/ctags.py b/pyls/plugins/ctags.py index 97165b32..18e59309 100644 --- a/pyls/plugins/ctags.py +++ b/pyls/plugins/ctags.py @@ -222,6 +222,7 @@ def parse_tags(tag_file, query): def parse_tag(line, query): match = TAG_RE.match(line) log.info("Got match %s from line: %s", match, line) + log.info("Line: ", line.replace('\t', '\\t').replace(' ', '\\s')) if not match: return None From 3e1d487f2b98c5c12e75de5a925104a1f49a29d7 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Sun, 17 Feb 2019 11:47:02 +0000 Subject: [PATCH 7/7] Test windows ctags --- pyls/plugins/ctags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyls/plugins/ctags.py b/pyls/plugins/ctags.py index 18e59309..e0b40595 100644 --- a/pyls/plugins/ctags.py +++ b/pyls/plugins/ctags.py @@ -222,7 +222,7 @@ def parse_tags(tag_file, query): def parse_tag(line, query): match = TAG_RE.match(line) log.info("Got match %s from line: %s", match, line) - log.info("Line: ", line.replace('\t', '\\t').replace(' ', '\\s')) + log.info("Line: %s", line.replace('\t', '\\t').replace(' ', '\\s')) if not match: return None