Add mypy support using daemon

.gitignore

@@ -109,3 +109,7 @@ ENV/
 
 # Merge orig files
 *.orig
+
+# Mypy
+.dmypy.json
+.mypy_cache/

pyls/plugins/mypy_lint.py

@@ -0,0 +1,158 @@
+# Copyright 2017 Palantir Technologies, Inc.
+import hashlib
+import logging
+import threading
+import re
+import sys
+import time
+
+from mypy import dmypy, dmypy_server, fscache, main, version
+
+from pyls import hookimpl, lsp, uris
+
+log = logging.getLogger(__name__)
+
+MYPY_RE = re.compile(r"([^:]+):(?:(\d+):)?(?:(\d+):)? (\w+): (.*)")
+
+
+@hookimpl
+def pyls_initialize(workspace):
+    log.info("Launching mypy server")
+    thread = threading.Thread(target=launch_daemon, args=([], workspace))
+    thread.daemon = True
+    thread.start()
+
+
+@hookimpl
+def pyls_lint(document):
+    args = _parse_daemon_args([document.path])
+    log.debug("Sending request to mypy daemon", args)
+    response = dmypy.request('run', version=version.__version__, args=args.flags)
+    log.debug("Got response from mypy daemon: %s", response)
+
+    # If the daemon signals that a restart is necessary, do it
+    if 'restart' in response:
+        # TODO(gatesn): figure out how to restart daemon
+        log.error("Need to restart daemon")
+        sys.exit("Need to restart mypy daemon")
+        # print('Restarting: {}'.format(response['restart']))
+        # restart_server(args, allow_sources=True)
+        # response = request('run', version=version.__version__, args=args.flags)
+
+    try:
+        stdout, stderr, status_code = response['out'], response['err'], response['status']
+        if stderr:
+            log.warning("Mypy stderr: %s", stderr)
+        return _process_mypy_output(stdout, document)
+    except KeyError:
+        log.error("Unknown mypy daemon response: %s", response)
+
+
+def _process_mypy_output(stdout, document):
+    for line in stdout.splitlines():
+        result = re.match(MYPY_RE, line)
+        if not result:
+            log.warning("Failed to parse mypy output: %s", line)
+            continue
+
+        _, lineno, offset, level, msg = result.groups()
+        lineno = (int(lineno) or 1) - 1
+        offset = (int(offset) or 1) - 1  # mypy says column numbers are zero-based, but they seem not to be
+
+        if level == "error":
+            severity = lsp.DiagnosticSeverity.Error
+        elif level == "warning":
+            severity = lsp.DiagnosticSeverity.Warning
+        elif level == "note":
+            severity = lsp.DiagnosticSeverity.Information
+        else:
+            log.warning("Unknown mypy severity: %s", level)
+            continue
+
+        diag = {
+            'source': 'mypy',
+            'range': {
+                'start': {'line': lineno, 'character': offset},
+                # There may be a better solution, but mypy does not provide end
+                'end': {'line': lineno, 'character': offset + 1}
+            },
+            'message': msg,
+            'severity': severity
+        }
+
+        # Try and guess the end of the word that mypy is highlighting
+        word = document.word_at_position(diag['range']['start'])
+        if word:
+            diag['range']['end']['character'] = offset + len(word)
+
+        yield diag
+
+
+def launch_daemon(raw_args, workspace):
+    """Launch the mypy daemon in-process."""
+    args = _parse_daemon_args(raw_args)
+    _sources, options = main.process_options(
+        ['-i'] + args.flags, require_targets=False, server_options=True
+    )
+    server = dmypy_server.Server(options)
+    server.fscache = PylsFileSystemCache(workspace)
+    server.serve()
+    log.error("mypy daemon stopped serving requests")
+
+
+def _parse_daemon_args(raw_args):
+    # TODO(gatesn): Take extra arguments from pyls config
+    return dmypy.parser.parse_args([
+        'run', '--',
+        '--show-traceback',
+        '--follow-imports=skip',
+        '--show-column-numbers',
+    ] + raw_args)
+
+
+class PylsFileSystemCache(fscache.FileSystemCache):
+    """Patched implementation of FileSystemCache to read from workspace."""
+
+    def __init__(self, workspace):
+        self._workspace = workspace
+        self._checksums = {}
+        self._mtimes = {}
+        super(PylsFileSystemCache, self).__init__()
+
+    def stat(self, path):
+        stat = super(PylsFileSystemCache, self).stat(path)
+
+        uri = uris.from_fs_path(path)
+        document = self._workspace.documents.get(uri)
+        if document:
+            size = len(document.source.encode('utf-8'))
+            mtime = self._workspace.get_document_mtime(uri)
+            log.debug("Patching os.stat response with size %s and mtime %s", size, mtime)
+            return MutableOsState(stat, {'st_size': size, 'st_mtime': mtime})
+
+        return stat
+
+    def read(self, path):
+        document = self._workspace.documents.get(uris.from_fs_path(path))
+        if document:
+            return document.source.encode('utf-8')  # Workspace returns unicode, we need bytes
+        return super(PylsFileSystemCache, self).read(path)
+
+    def md5(self, path):
+        document = self._workspace.documents.get(uris.from_fs_path(path))
+        if document:
+            return hashlib.md5(document.source.encode('utf-8')).hexdigest()
+        return super(PylsFileSystemCache, self).read(path)
+
+
+class MutableOsState(object):
+    """Wrapper around a stat_result that allows us to override values."""
+
+    def __init__(self, stat_result, overrides):
+        self._stat_result = stat_result
+        self._overrides = overrides
+
+    def __getattr__(self, item):
+        if item in self._overrides:
+            return self._overrides[item]
+        return getattr(self._stat_result, item)

pyls/workspace.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import re
+import time
 
 import jedi
 
@@ -27,6 +28,7 @@ def __init__(self, root_uri, endpoint):
         self._root_uri_scheme = uris.urlparse(self._root_uri)[0]
         self._root_path = uris.to_fs_path(self._root_uri)
         self._docs = {}
+        self._doc_mtimes = {}
 
         # Whilst incubating, keep rope private
         self.__rope = None
@@ -69,13 +71,19 @@ def get_document(self, doc_uri):
 
     def put_document(self, doc_uri, source, version=None):
         self._docs[doc_uri] = self._create_document(doc_uri, source=source, version=version)
+        self._doc_mtimes[doc_uri] = time.time()
 
     def rm_document(self, doc_uri):
         self._docs.pop(doc_uri)
 
     def update_document(self, doc_uri, change, version=None):
         self._docs[doc_uri].apply_change(change)
         self._docs[doc_uri].version = version
+        self._doc_mtimes[doc_uri] = time.time()
+
+    def get_document_mtime(self, doc_uri):
+        # TODO(gatesn): Do we want to fall back to os.stat?
+        return self._doc_mtimes[doc_uri]
 
     def apply_edit(self, edit):
         return self._endpoint.request(self.M_APPLY_EDIT, {'edit': edit})

setup.py

@@ -48,6 +48,7 @@
         'all': [
             'autopep8',
             'mccabe',
+            'mypy; python_version>="3.4"',
             'pycodestyle',
             'pydocstyle>=2.0.0',
             'pyflakes>=1.6.0',
@@ -56,6 +57,7 @@
         ],
         'autopep8': ['autopep8'],
         'mccabe': ['mccabe'],
+        'mypy': ['mypy; python_version>="3.4"'],
         'pycodestyle': ['pycodestyle'],
         'pydocstyle': ['pydocstyle>=2.0.0'],
         'pyflakes': ['pyflakes>=1.6.0'],
@@ -81,6 +83,7 @@
             'jedi_signature_help = pyls.plugins.signature',
             'jedi_symbols = pyls.plugins.symbols',
             'mccabe = pyls.plugins.mccabe_lint',
+            'mypy = pyls.plugins.mypy_lint',
             'preload = pyls.plugins.preload_imports',
             'pycodestyle = pyls.plugins.pycodestyle_lint',
             'pydocstyle = pyls.plugins.pydocstyle_lint',

vscode-client/src/extension.ts

@@ -43,7 +43,7 @@ function startLangServerTCP(addr: number, documentSelector: string[]): Disposabl
 }
 
 export function activate(context: ExtensionContext) {
-    context.subscriptions.push(startLangServer("pyls", ["-vv"], ["python"]));
+    context.subscriptions.push(startLangServer("pyls", ["-v"], ["python"]));
     // For TCP server needs to be started seperately
     // context.subscriptions.push(startLangServerTCP(2087, ["python"]));
 }

vscode-client/yarn.lock

@@ -223,10 +223,6 @@ clone@^1.0.0:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.3.tgz#298d7e2231660f40c003c2ed3140decf3f53085f"
 
-clone@^2.1.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.1.tgz#d217d1e961118e3ac9a4b8bba3285553bf647cdb"
-
 cloneable-readable@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/cloneable-readable/-/cloneable-readable-1.0.0.tgz#a6290d413f217a61232f95e458ff38418cfb0117"
@@ -253,18 +249,14 @@ [email protected]:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/commander/-/commander-0.6.1.tgz#fa68a14f6a945d54dbbe50d8cdb3320e9e3b1a06"
 
-[email protected]:
+[email protected], commander@^2.9.0:
   version "2.11.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563"
 
 [email protected]:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.3.0.tgz#fd430e889832ec353b9acd1de217c11cb3eef873"
 
-commander@^2.9.0:
-  version "2.13.0"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.13.0.tgz#6964bca67685df7c1f1430c584f07d7597885b9c"
-
 [email protected]:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -414,14 +406,10 @@ extglob@^0.3.1:
   dependencies:
     is-extglob "^1.0.0"
 
-[email protected]:
+[email protected], extsprintf@^1.2.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05"
 
-extsprintf@^1.2.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f"
-
 fancy-log@^1.1.0:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/fancy-log/-/fancy-log-1.3.2.tgz#f41125e3d84f2e7d89a43d06d958c8f78be16be1"
@@ -1736,18 +1724,7 @@ vinyl@^1.0.0:
     clone-stats "^0.0.1"
     replace-ext "0.0.1"
 
-vinyl@^2.0.2:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/vinyl/-/vinyl-2.1.0.tgz#021f9c2cf951d6b939943c89eb5ee5add4fd924c"
-  dependencies:
-    clone "^2.1.1"
-    clone-buffer "^1.0.0"
-    clone-stats "^1.0.0"
-    cloneable-readable "^1.0.0"
-    remove-trailing-separator "^1.0.1"
-    replace-ext "^1.0.0"
-
-vinyl@~2.0.1:
+vinyl@^2.0.2, vinyl@~2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/vinyl/-/vinyl-2.0.2.tgz#0a3713d8d4e9221c58f10ca16c0116c9e25eda7c"
   dependencies: