From 01e6b34c8d4f129f0872bcdc5aebe7789b74006e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 6 Jun 2024 21:03:47 -0700 Subject: [PATCH] Add initial pyright support Refs #785. --- DOCS.md | 20 ++++-- Makefile | 8 +++ coconut/command/cli.py | 6 ++ coconut/command/command.py | 68 ++++++++++++-------- coconut/command/resources/pyrightconfig.json | 7 -- coconut/command/util.py | 44 ++++++++++--- coconut/constants.py | 12 ++++ coconut/requirements.py | 1 + coconut/root.py | 2 +- 9 files changed, 118 insertions(+), 50 deletions(-) delete mode 100644 coconut/command/resources/pyrightconfig.json diff --git a/DOCS.md b/DOCS.md index cb85a868c..a341d3a4a 100644 --- a/DOCS.md +++ b/DOCS.md @@ -92,6 +92,7 @@ The full list of optional dependencies is: - `kernel`: lightweight subset of `jupyter` that only includes the dependencies that are strictly necessary for Coconut's [Jupyter kernel](#kernel). - `watch`: enables use of the `--watch` flag. - `mypy`: enables use of the `--mypy` flag. +- `pyright`: enables use of the `--pyright` flag. - `xonsh`: enables use of Coconut's [`xonsh` support](#xonsh-support). - `numpy`: installs everything necessary for making use of Coconut's [`numpy` integration](#numpy-integration). - `jupyterlab`: installs everything necessary to use [JupyterLab](https://github.com/jupyterlab/jupyterlab) with Coconut. @@ -121,11 +122,11 @@ depth: 1 ``` coconut [-h] [--and source [dest ...]] [-v] [-t version] [-i] [-p] [-a] [-l] - [--no-line-numbers] [-k] [-w] [-r] [-n] [-d] [-q] [-s] [--no-tco] - [--no-wrap-types] [-c code] [--incremental] [-j processes] [-f] [--minify] - [--jupyter ...] [--mypy ...] [--argv ...] [--tutorial] [--docs] [--style name] - [--vi-mode] [--recursion-limit limit] [--stack-size kbs] [--site-install] - [--site-uninstall] [--verbose] [--trace] [--profile] + [--no-line-numbers] [-k] [-w] [-r] [-n] [-d] [-q] [-s] [--no-tco] [--no-wrap-types] + [-c code] [-j processes] [-f] [--minify] [--jupyter ...] [--mypy ...] [--pyright] + [--argv ...] [--tutorial] [--docs] [--style name] [--vi-mode] + [--recursion-limit limit] [--stack-size kbs] [--fail-fast] [--no-cache] + [--site-install] [--site-uninstall] [--verbose] [--trace] [--profile] [source] [dest] ``` @@ -184,6 +185,7 @@ dest destination directory for compiled files (defaults to Jupyter) --mypy ... run MyPy on compiled Python (remaining args passed to MyPy) (implies --package --line-numbers) +--pyright run Pyright on compiled Python (implies --package) --argv ..., --args ... set sys.argv to source plus remaining args for use in the Coconut script being run @@ -452,6 +454,10 @@ You can also run `mypy`—or any other static type checker—directly on the com To distribute your code with checkable type annotations, you'll need to include `coconut` as a dependency (though a `--no-deps` install should be fine), as installing it is necessary to make the requisite stub files available. You'll also probably want to include a [`py.typed`](https://peps.python.org/pep-0561/) file. +##### Pyright Integration + +Though not as well-supported as MyPy, Coconut also has built-in [Pyright](https://github.com/microsoft/pyright) support. Simply pass `--pyright` to automatically run Pyright on all compiled code. To adjust Pyright options, rather than pass them at the command-line, add your settings to the file `~/.coconut_pyrightconfig.json` (automatically generated the first time `coconut --pyright` is run). + ##### Syntax To explicitly annotate your code with types to be checked, Coconut supports (on all Python versions): @@ -467,7 +473,7 @@ Sometimes, MyPy will not know how to handle certain Coconut constructs, such as ##### Interpreter -Coconut even supports `--mypy` in the interpreter, which will intelligently scan each new line of code, in the context of previous lines, for newly-introduced MyPy errors. For example: +Coconut even supports `--mypy` (though not `--pyright`) in the interpreter, which will intelligently scan each new line of code, in the context of previous lines, for newly-introduced MyPy errors. For example: ```coconut_pycon >>> a: str = count()[0] :14: error: Incompatible types in assignment (expression has type "int", variable has type "str") @@ -4655,7 +4661,7 @@ else: #### `reveal_type` and `reveal_locals` -When using MyPy, `reveal_type()` will cause MyPy to print the type of `` and `reveal_locals()` will cause MyPy to print the types of the current `locals()`. At runtime, `reveal_type(x)` is always the identity function and `reveal_locals()` always returns `None`. See [the MyPy documentation](https://mypy.readthedocs.io/en/stable/common_issues.html#reveal-type) for more information. +When using static type analysis tools integrated with Coconut such as [MyPy](#mypy-integration), `reveal_type()` will cause MyPy to print the type of `` and `reveal_locals()` will cause MyPy to print the types of the current `locals()`. At runtime, `reveal_type(x)` is always the identity function and `reveal_locals()` always returns `None`. See [the MyPy documentation](https://mypy.readthedocs.io/en/stable/common_issues.html#reveal-type) for more information. ##### Example diff --git a/Makefile b/Makefile index afa7695a6..e96ed3eef 100644 --- a/Makefile +++ b/Makefile @@ -161,6 +161,14 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py +# same as test-mypy but uses pyright instead +.PHONY: test-pyright +test-pyright: export COCONUT_USE_COLOR=TRUE +test-pyright: clean + python ./coconut/tests --strict --keep-lines --force --target sys --no-cache --pyright + python ./coconut/tests/dest/runner.py + python ./coconut/tests/dest/extras.py + # same as test-univ but includes verbose output for better debugging # regex for getting non-timing lines: ^(?!'|\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed|Incremental|Pruned|Compiled)\s)[^\n]*\n* .PHONY: test-verbose diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 5ea28e199..c542cab6f 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -216,6 +216,12 @@ help="run MyPy on compiled Python (remaining args passed to MyPy) (implies --package --line-numbers)", ) +arguments.add_argument( + "--pyright", + action="store_true", + help="run Pyright on compiled Python (implies --package)", +) + arguments.add_argument( "--argv", "--args", type=str, diff --git a/coconut/command/command.py b/coconut/command/command.py index f6596be1c..6c6ca5a45 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -83,8 +83,6 @@ first_import_time, ) from coconut.command.util import ( - writefile, - readfile, showpath, rem_encoding, Runner, @@ -104,6 +102,7 @@ run_with_stack_size, proc_run_args, get_python_lib, + update_pyright_config, ) from coconut.compiler.util import ( should_indent, @@ -128,6 +127,7 @@ class Command(object): display = False # corresponds to --display flag jobs = 0 # corresponds to --jobs flag mypy_args = None # corresponds to --mypy flag + pyright = False # corresponds to --pyright flag argv_args = None # corresponds to --argv flag stack_size = 0 # corresponds to --stack-size flag use_cache = USE_CACHE # corresponds to --no-cache flag @@ -252,6 +252,8 @@ def execute_args(self, args, interact=True, original_args=None): logger.log("Directly passed args:", original_args) logger.log("Parsed args:", args) + type_checking_arg = "--mypy" if args.mypy else "--pyright" if args.pyright else None + # validate args and show warnings if args.stack_size and args.stack_size % 4 != 0: logger.warn("--stack-size should generally be a multiple of 4, not {stack_size} (to support 4 KB pages)".format(stack_size=args.stack_size)) @@ -259,8 +261,8 @@ def execute_args(self, args, interact=True, original_args=None): logger.warn("using --mypy running with --no-line-numbers is not recommended; mypy error messages won't include Coconut line numbers") if args.interact and args.run: logger.warn("extraneous --run argument passed; --interact implies --run") - if args.package and self.mypy: - logger.warn("extraneous --package argument passed; --mypy implies --package") + if args.package and type_checking_arg: + logger.warn("extraneous --package argument passed; --{type_checking_arg} implies --package".format(type_checking_arg=type_checking_arg)) # validate args and raise errors if args.line_numbers and args.no_line_numbers: @@ -269,10 +271,10 @@ def execute_args(self, args, interact=True, original_args=None): raise CoconutException("cannot --site-install and --site-uninstall simultaneously") if args.standalone and args.package: raise CoconutException("cannot compile as both --package and --standalone") - if args.standalone and self.mypy: - raise CoconutException("cannot compile as both --package (implied by --mypy) and --standalone") - if args.no_write and self.mypy: - raise CoconutException("cannot compile with --no-write when using --mypy") + if args.standalone and type_checking_arg: + raise CoconutException("cannot compile as both --package (implied by --{type_checking_arg}) and --standalone".format(type_checking_arg=type_checking_arg)) + if args.no_write and type_checking_arg: + raise CoconutException("cannot compile with --no-write when using --{type_checking_arg}".format(type_checking_arg=type_checking_arg)) for and_args in getattr(args, "and") or []: if len(and_args) > 2: raise CoconutException( @@ -291,6 +293,7 @@ def execute_args(self, args, interact=True, original_args=None): set_recursion_limit(args.recursion_limit) self.fail_fast = args.fail_fast self.display = args.display + self.pyright = args.pyright self.prompt.vi_mode = args.vi_mode if args.style is not None: self.prompt.set_style(args.style) @@ -375,8 +378,8 @@ def execute_args(self, args, interact=True, original_args=None): for kwargs in all_compile_path_kwargs: filepaths += self.compile_path(**kwargs) - # run mypy on compiled files - self.run_mypy(filepaths) + # run type checking on compiled files + self.run_type_checking(filepaths) # do extra compilation if there is any if extra_compile_path_kwargs: @@ -456,7 +459,7 @@ def process_source_dest(self, source, dest, args): processed_dest = dest # determine package mode - if args.package or self.mypy: + if args.package or self.type_checking: package = True elif args.standalone: package = False @@ -576,7 +579,7 @@ def compile_file(self, filepath, write=True, package=False, force=False, **kwarg def compile(self, codepath, destpath=None, package=False, run=False, force=False, show_unchanged=True, handling_exceptions_kwargs={}, callback=None): """Compile a source Coconut file to a destination Python file.""" with univ_open(codepath, "r") as opened: - code = readfile(opened) + code = opened.read() package_level = -1 if destpath is not None: @@ -607,7 +610,7 @@ def inner_callback(compiled): logger.show_tabulated("Compiled", showpath(codepath), "without writing to file.") else: with univ_open(destpath, "w") as opened: - writefile(opened, compiled) + opened.write(compiled) logger.show_tabulated("Compiled to", showpath(destpath), ".") if self.display: logger.print(compiled) @@ -657,7 +660,7 @@ def create_package(self, dirpath, retries_left=create_package_retries): filepath = os.path.join(dirpath, "__coconut__.py") try: with univ_open(filepath, "w") as opened: - writefile(opened, self.comp.getheader("__coconut__")) + opened.write(self.comp.getheader("__coconut__")) except OSError: logger.log_exc() if retries_left <= 0: @@ -792,7 +795,7 @@ def has_hash_of(self, destpath, code, package_level): """Determine if a file has the hash of the code.""" if destpath is not None and os.path.isfile(destpath): with univ_open(destpath, "r") as opened: - compiled = readfile(opened) + compiled = opened.read() hashash = gethash(compiled) if hashash is not None: newhash = self.comp.genhash(code, package_level) @@ -880,7 +883,7 @@ def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): logger.print(compiled) if path is None: # header is not included - if not self.mypy: + if not self.type_checking: no_str_code = self.comp.remove_strs(compiled) if no_str_code is not None: result = mypy_builtin_regex.search(no_str_code) @@ -892,7 +895,7 @@ def execute(self, compiled=None, path=None, use_eval=False, allow_show=True): self.runner.run(compiled, use_eval=use_eval, path=path, all_errors_exit=path is not None) - self.run_mypy(code=self.runner.was_run_code()) + self.run_type_checking(code=self.runner.was_run_code()) def execute_file(self, destpath, **kwargs): """Execute compiled file.""" @@ -912,15 +915,20 @@ def check_runner(self, set_sys_vars=True, argv_source_path=""): # set up runner if self.runner is None: - self.runner = Runner(self.comp, exit=self.exit_runner, store=self.mypy) + self.runner = Runner(self.comp, exit=self.exit_runner, store=self.type_checking) # pass runner to prompt self.prompt.set_runner(self.runner) @property - def mypy(self): - """Whether using MyPy or not.""" - return self.mypy_args is not None + def type_checking(self): + """Whether using a static type-checker or not.""" + return self.mypy_args is not None or self.pyright + + @property + def type_checking_version(self): + """What version of Python to type check against.""" + return ver_tuple_to_str(get_target_info_smart(self.comp.target, mode="highest")) def set_mypy_args(self, mypy_args=None): """Set MyPy arguments.""" @@ -940,7 +948,7 @@ def set_mypy_args(self, mypy_args=None): if not any(arg.startswith("--python-version") for arg in self.mypy_args): self.mypy_args += [ "--python-version", - ver_tuple_to_str(get_target_info_smart(self.comp.target, mode="highest")), + self.type_checking_version, ] if not any(arg.startswith("--python-executable") for arg in self.mypy_args): @@ -960,9 +968,9 @@ def set_mypy_args(self, mypy_args=None): logger.log("MyPy args:", self.mypy_args) self.mypy_errs = [] - def run_mypy(self, paths=(), code=None): - """Run MyPy with arguments.""" - if self.mypy: + def run_type_checking(self, paths=(), code=None): + """Run type-checking on the given paths / code.""" + if self.mypy_args is not None: set_mypy_path() from coconut.command.mypy import mypy_run args = list(paths) + self.mypy_args @@ -987,6 +995,14 @@ def run_mypy(self, paths=(), code=None): if code is not None: # interpreter logger.printerr(line) self.mypy_errs.append(line) + if self.pyright: + config_file = update_pyright_config() + if code is not None: + logger.warn("--pyright only works on files, not code snippets or at the interpreter") + if paths: + from pyright import main + args = ["--project", config_file, "--pythonversion", self.type_checking_version] + list(paths) + main(args) def run_silent_cmd(self, *args): """Same as run_cmd$(show_output=logger.verbose).""" @@ -1157,7 +1173,7 @@ def error_callback(err): writedir = os.path.join(dest, os.path.relpath(dirpath, src)) def inner_callback(path): - self.run_mypy([path]) + self.run_type_checking([path]) callback() self.compile_path( path, diff --git a/coconut/command/resources/pyrightconfig.json b/coconut/command/resources/pyrightconfig.json deleted file mode 100644 index 07d25add6..000000000 --- a/coconut/command/resources/pyrightconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extraPaths": [ - "C://Users/evanj/.coconut_stubs" - ], - "pythonVersion": "3.11", - "reportPossiblyUnboundVariable": false -} diff --git a/coconut/command/util.py b/coconut/command/util.py index c4e0b1e7d..ccd11d74e 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -24,6 +24,7 @@ import subprocess import shutil import threading +import json from select import select from contextlib import contextmanager from functools import partial @@ -50,6 +51,7 @@ get_encoding, get_clock_time, assert_remove_prefix, + univ_open, ) from coconut.constants import ( WINDOWS, @@ -88,6 +90,9 @@ high_proc_prio, call_timeout, use_fancy_call_output, + extra_pyright_args, + pyright_config_file, + tabideal, ) if PY26: @@ -148,17 +153,23 @@ # ----------------------------------------------------------------------------------------------------------------------- -def writefile(openedfile, newcontents): - """Set the contents of a file.""" +def writefile(openedfile, newcontents, in_json=False, **kwargs): + """Set the entire contents of a file regardless of current position.""" openedfile.seek(0) openedfile.truncate() - openedfile.write(newcontents) + if in_json: + json.dump(newcontents, openedfile, **kwargs) + else: + openedfile.write(newcontents, **kwargs) -def readfile(openedfile): - """Read the contents of a file.""" +def readfile(openedfile, in_json=False, **kwargs): + """Read the entire contents of a file regardless of current position.""" openedfile.seek(0) - return str(openedfile.read()) + if in_json: + return json.load(openedfile, **kwargs) + else: + return str(openedfile.read(**kwargs)) def open_website(url): @@ -450,8 +461,8 @@ def symlink(link_to, link_from): shutil.copytree(link_to, link_from) -def install_mypy_stubs(): - """Properly symlink mypy stub files.""" +def install_stubs(): + """Properly symlink stub files for type-checking purposes.""" # unlink stub_dirs so we know rm_dir_or_link won't clear them for stub_name in stub_dir_names: unlink(os.path.join(base_stub_dir, stub_name)) @@ -480,7 +491,7 @@ def set_env_var(name, value): def set_mypy_path(): """Put Coconut stubs in MYPYPATH.""" # mypy complains about the path if we don't use / over \ - install_dir = install_mypy_stubs().replace(os.sep, "/") + install_dir = install_stubs().replace(os.sep, "/") original = os.getenv(mypy_path_env_var) if original is None: new_mypy_path = install_dir @@ -494,6 +505,21 @@ def set_mypy_path(): return install_dir +def update_pyright_config(python_version=None): + """Save an updated pyrightconfig.json.""" + update_existing = os.path.exists(pyright_config_file) + with univ_open(pyright_config_file, "r+" if update_existing else "w") as config_file: + if update_existing: + config = readfile(config_file, in_json=True) + else: + config = extra_pyright_args.copy() + config["extraPaths"] = [install_stubs()] + if python_version is not None: + config["pythonVersion"] = python_version + writefile(config_file, config, in_json=True, indent=tabideal) + return pyright_config_file + + def is_empty_pipe(pipe, default=None): """Determine if the given pipe file object is empty.""" if pipe.closed: diff --git a/coconut/constants.py b/coconut/constants.py index 30439cd7c..522faeec0 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -670,6 +670,8 @@ def get_path_env_var(env_var, default): ) installed_stub_dir = os.path.join(coconut_home, ".coconut_stubs") +pyright_config_file = os.path.join(coconut_home, ".coconut_pyrightconfig.json") + watch_interval = .1 # seconds info_tabulation = 18 # offset for tabulated info messages @@ -722,6 +724,10 @@ def get_path_env_var(env_var, default): ": note: ", ) +extra_pyright_args = { + "reportPossiblyUnboundVariable": False, +} + oserror_retcode = 127 kilobyte = 1024 @@ -985,6 +991,11 @@ def get_path_env_var(env_var, default): "types-backports", ("typing", "py<35"), ), + "pyright": ( + "pyright", + "types-backports", + ("typing", "py<35"), + ), "watch": ( "watchdog", ), @@ -1041,6 +1052,7 @@ def get_path_env_var(env_var, default): "myst-parser": (3,), "sphinx": (7,), "mypy[python2]": (1, 10), + "pyright": (1, 1), ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), ("typing_extensions", "py>=38"): (4, 11), diff --git a/coconut/requirements.py b/coconut/requirements.py index 05be7b6d4..c6db84597 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -223,6 +223,7 @@ def everything_in(req_dict): "kernel": get_reqs("kernel"), "watch": get_reqs("watch"), "mypy": get_reqs("mypy"), + "pyright": get_reqs("pyright"), "xonsh": get_reqs("xonsh"), "numpy": get_reqs("numpy"), } diff --git a/coconut/root.py b/coconut/root.py index 3b7a1c2e9..ea832907c 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.1.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 15 +DEVELOP = 16 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1"