From f70cd4a5a1ad58ae62479265d50a788c450509f6 Mon Sep 17 00:00:00 2001 From: Yuki Igarashi Date: Fri, 12 Nov 2021 17:33:27 +0900 Subject: [PATCH] Sync with internal repository Thank you for all your contributions. Contributions: Yuki Igarashi: 14 commits, 431 lines Ryo Miyajima: 5 commits, 296 lines git-pysen-release-hash: 1baa6934e91ffee1763013c53738aa749a3ceb29 Co-authored-by: Yuki Igarashi Co-authored-by: Ryo Miyajima --- README.md | 110 +++++++++++++++++++++++----------- pyproject.toml | 4 +- pysen/__main__.py | 4 ++ pysen/_version.py | 2 +- pysen/cli.py | 2 +- pysen/command.py | 4 +- pysen/ext/black_wrapper.py | 16 +++-- pysen/ext/flake8_wrapper.py | 12 ++-- pysen/ext/isort_wrapper.py | 6 +- pysen/ext/mypy_wrapper.py | 24 ++++++-- pysen/process_utils.py | 29 +++++++-- pysen/py_version.py | 11 ++++ pysen/pyproject.py | 42 +++++++++---- pysen/pyproject_model.py | 6 +- setup.py | 2 +- tests/test_mypy.py | 5 +- tests/test_process_utils.py | 31 +++++++++- tests/test_py_version.py | 20 +++++++ tests/test_pyproject.py | 82 ++++++++++++++++++++++++- tests/test_pyproject_model.py | 4 ++ tox.ini | 15 ++++- 21 files changed, 342 insertions(+), 89 deletions(-) create mode 100644 pysen/__main__.py diff --git a/README.md b/README.md index 7938e5b..a7f49c3 100644 --- a/README.md +++ b/README.md @@ -2,57 +2,40 @@ ![](https://github.com/pfnet/pysen/blob/main/assets/imgs/pysen.gif?raw=true) -## What is pysen? - -pysen aims to provide a unified platform to configure and run day-to-day development tools. -We envision the following scenarios in the future: - -- You open any project and `pysen run lint`, `pysen run format` will check and format the entire codebase -- Standardized coding styles are setup with a few lines in a single `pyproject.toml` file - -pysen centralizes the code and knowledge related to development tools that teams have accumulated, most notably for python linters. -You can make tasks that can be executed from both `setup.py` and our command-line tool. -We currently provide tasks that manage setting files for the following tools: - -- linters - - flake8 - - isort - - mypy - - black -- utilities - - (planned) protoc - -## What isn't pysen? - -* pysen is not a linting tool per se. Rather, `pysen run lint` orchestrates multiple python linting tools by automatically setting up their configurations from a more abstract setting for pysen. -* pysen does not manage your depedencies and packages. We recommend using package managers such as [pipenv](https://github.com/pypa/pipenv) or [poetry](https://python-poetry.org/) to lock your dependecy versions, **including the versions for the linting tools that pysen coordinates** (i.e., isort, mypy, flake8, black). The supported versions for these tools can be found in the `extra_requires/lint` section in pysen's [setup.py](https://github.com/pfnet/pysen/blob/main/setup.py). You should **not** rely on `pip install pysen[lint]` to control the versions of your linting tools. -* pysen is not limited to linting purposes or python. See the [plugin section](README.md#create-a-plugin-to-customize-pysen) for details. - ## Install ### PyPI +#### If you have no preference of linter versions (recommended for newbies) + ```sh pip install "pysen[lint]" ``` +#### Install pysen with your choice of linter versions + +```sh +pip install pysen +pip install black==21.10b0 flake8==4.0.1 isort==5.10.1 mypy==0.910 +``` + ### Other installation examples ```sh # pipenv -pipenv install --dev "pysen[lint]==0.9.1" +pipenv install --dev "pysen[lint]==0.10.1" # poetry -poetry add -D pysen==0.9.1 -E lint +poetry add -D pysen==0.10.1 -E lint ``` ## Quickstart: Set up linters using pysen -Put the following pysen configuration to `pyproject.toml` of your python package: +Put the following pysen configuration to either `pysen.toml` or `pyproject.toml` of your python package: ```toml [tool.pysen] -version = "0.9" +version = "0.10" [tool.pysen.lint] enable_black = true @@ -75,7 +58,7 @@ $ pysen run format # corrects errors with compatible commands (black, isort) That's it! pysen, or more accurately pysen tasks that support the specified linters, generate setting files for black, isort, mypy, and flake8 and run them with the appropriate configuration. -For more details about the configuration items that you can write in `pyproject.toml`, please refer to `pysen/pyproject_model.py`. +For more details about the configuration items that you can write in a config file, please refer to `pysen/pyproject_model.py`. You can also add custom setup commands to your Python package by adding the following lines to its `setup.py`: ```py @@ -92,6 +75,56 @@ For more details, please refer to the following two examples: - Example configuration from Python: `examples/advanced_example/config.py` - Example plugin for pysen: `examples/plugin_example/plugin.py` +## Frequently Asked Questions + +Q. How do I use `mypy >= 0.800`? +A. See [Install pysen with your choice of linter versions](#install-pysen-with-your-choice-of-linter-versions) + +Q. mypy reports the error `Source file found twice under different module names`. +A. Add `tool.pysen.lint.mypy_targets` section(s) so file names are unique in each section. + +Q. How do I change specific settings for linter X? +A. We prioritize convention over configuration. However you can always create your own plugin. See: [Create a plugin to customize pysen](#create-a-plugin-to-customize-pysen) + +Q. pysen seems to ignore some files. +A. pysen only checks files that are tracked in git. Try `git add`ing the file under question. +You can also disable this behavior by setting the environment variable `PYSEN_IGNORE_GIT=1`. + +Q. How do I run only [flake8|black|isort|mypy]? +A. Try the `--enable` and `--disable` options, for example, `pysen --enable flake --enable black run lint`. + +Q. Files without filename extensions are not checked. +A. Explicitly add those files under the include section in `tool.pysen.lint.source`. + +Q. How do I add additional settings to my `pyproject.toml`, e.g., [pydantic-mypy](https://pydantic-docs.helpmanual.io/mypy_plugin/#configuring-the-plugin)? +A. Add `settings_dir="."` under the `[tool.pysen-cli]` section. + +## What is pysen? + +pysen aims to provide a unified platform to configure and run day-to-day development tools. +We envision the following scenarios in the future: + +- You open any project and `pysen run lint`, `pysen run format` will check and format the entire codebase +- Standardized coding styles are setup with a few lines in a single config file + +pysen centralizes the code and knowledge related to development tools that teams have accumulated, most notably for python linters. +You can make tasks that can be executed from both `setup.py` and our command-line tool. +We currently provide tasks that manage setting files for the following tools: + +- linters + - flake8 + - isort + - mypy + - black +- utilities + - (planned) protoc + +## What isn't pysen? + +* pysen is not a linting tool per se. Rather, `pysen run lint` orchestrates multiple python linting tools by automatically setting up their configurations from a more abstract setting for pysen. +* pysen does not manage your depedencies and packages. We recommend using package managers such as [pipenv](https://github.com/pypa/pipenv) or [poetry](https://python-poetry.org/) to lock your dependecy versions, **including the versions for the linting tools that pysen coordinates** (i.e., isort, mypy, flake8, black). The supported versions for these tools can be found in the `extra_requires/lint` section in pysen's [setup.py](https://github.com/pfnet/pysen/blob/main/setup.py). You should **not** rely on `pip install pysen[lint]` to control the versions of your linting tools. +* pysen is not limited to linting purposes or python. See the [plugin section](README.md#create-a-plugin-to-customize-pysen) for details. + ## How it works: Settings file directory Under the hood, whenever you run pysen, it generates the setting files as ephemeral temporary files to be used by linters. @@ -103,7 +136,7 @@ $ pysen generate [out_dir] ``` You can specify the settings directory that pysen uses when you `pysen run`. -To do so add the following section to your `pyproject.toml`: +To do so add the following section to your config: ```toml [tool.pysen-cli] @@ -137,6 +170,9 @@ The result will look like the following: ![pysen-vim](https://github.com/pfnet/pysen/blob/main/assets/imgs/pysen_vim.gif?raw=true) +A third party plugin is also available. +- [pysen.vim](https://github.com/bonprosoft/pysen.vim) + ### Emacs Refer to the [Compilation mode](https://www.gnu.org/software/emacs/manual/html_node/emacs/Compilation-Mode.html). @@ -158,21 +194,21 @@ Note that this may report duplicate errors if you have configured linters like ` We provide two methods to write configuration for pysen. -One is the `[tool.pysen.lint]` section in `pyproject.toml`. +One is the `[tool.pysen.lint]` section in the config. It is the most simple way to configure pysen, but the settings we provide are limited. The other method is to write a python script that configures pysen directly. If you want to customize configuration files that pysen generates, command-line arguments that pysen takes, or whatever actions pysen performs, we recommend you use this method. For more examples, please refer to `pysen/examples`. -### pyproject.toml configuration model +### Configuration model Please refer to `pysen/pyproject_model.py` for the latest model. Here is an example of a basic configuration: ```toml [tool.pysen] -version = "0.9" +version = "0.10" [tool.pysen.lint] enable_black = true @@ -202,6 +238,10 @@ mypy_path = ["stubs"] ignore_errors = true ``` +pysen looks for a configuration file in the following order: +1. `pysen.toml` with a `tool.pysen` section +2. `pyproject.toml` with a `tool.pysen` section + ### Create a plugin to customize pysen We provide a plugin interface for customizing our tool support, setting files management, setup commands and so on. diff --git a/pyproject.toml b/pyproject.toml index 92524a9..dfef163 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,14 @@ settings_dir = "." [tool.pysen] -version = "0.9" +version = "0.10" [tool.pysen.lint] enable_black = true enable_flake8 = true enable_isort = true enable_mypy = true -mypy_preset = "strict" +mypy_preset = "very_strict" py_version = "py37" isort_known_first_party = ["fakes", "pysen"] [[tool.pysen.lint.mypy_targets]] diff --git a/pysen/__main__.py b/pysen/__main__.py new file mode 100644 index 0000000..98dcca0 --- /dev/null +++ b/pysen/__main__.py @@ -0,0 +1,4 @@ +from .cli import cli + +if __name__ == "__main__": + cli() diff --git a/pysen/_version.py b/pysen/_version.py index d69d16e..1f4c4d4 100644 --- a/pysen/_version.py +++ b/pysen/_version.py @@ -1 +1 @@ -__version__ = "0.9.1" +__version__ = "0.10.1" diff --git a/pysen/cli.py b/pysen/cli.py index 72bc101..6680dda 100644 --- a/pysen/cli.py +++ b/pysen/cli.py @@ -223,7 +223,7 @@ def _setup_manifest_parser() -> argparse.ArgumentParser: parser.add_argument( "--config", type=str, - help="Path for pyproject.toml", + help="Path to a config file", default=None, ) parser.add_argument( diff --git a/pysen/command.py b/pysen/command.py index 79bac7d..339ee62 100644 --- a/pysen/command.py +++ b/pysen/command.py @@ -28,7 +28,9 @@ def run_files(self, reporter: Reporter, files: Sequence[pathlib.Path]) -> int: raise RunTargetFileNotSupported(self.name) -def check_command_installed(*validation_command: str) -> None: +def check_command_installed( + *validation_command: str, +) -> None: err = CommandNotFoundError( f"The command `{' '.join(validation_command)}` failed." " Make sure it is installed." diff --git a/pysen/ext/black_wrapper.py b/pysen/ext/black_wrapper.py index 52854d1..b4f4f4d 100644 --- a/pysen/ext/black_wrapper.py +++ b/pysen/ext/black_wrapper.py @@ -54,15 +54,11 @@ def _parse_file_path(file_path: str) -> pathlib.Path: @functools.lru_cache(1) def _check_black_version() -> None: version = get_version("black") - compatible_versions = [ - VersionRepresentation(19, 10), - VersionRepresentation(20, 8), - ] + minimum_supported = VersionRepresentation(19, 10) - if all(not v.is_compatible(version) for v in compatible_versions): + if version < minimum_supported: raise IncompatibleVersionError( - "pysen only supports black versions: " - f"{{{', '.join(v.version for v in compatible_versions)}}}. " + f"pysen only supports black >= {minimum_supported}, " f"version {version} is not supported." ) @@ -74,7 +70,7 @@ def run( sources: Iterable[pathlib.Path], inplace_edit: bool, ) -> int: - check_command_installed("black", "--version") + check_command_installed(*process_utils.add_python_executable("black", "--version")) _check_black_version() targets = [str(d) for d in sources] @@ -87,7 +83,9 @@ def run( + targets ) with change_dir(base_dir): - ret, stdout, _ = process_utils.run(cmd, reporter) + ret, stdout, _ = process_utils.run( + process_utils.add_python_executable(*cmd), reporter + ) diagnostics = parse_error_diffs(stdout, _parse_file_path, logger=reporter.logger) reporter.report_diagnostics(list(diagnostics)) diff --git a/pysen/ext/flake8_wrapper.py b/pysen/ext/flake8_wrapper.py index ebeedb8..516c9a2 100644 --- a/pysen/ext/flake8_wrapper.py +++ b/pysen/ext/flake8_wrapper.py @@ -10,6 +10,7 @@ from pysen.error_lines import parse_error_lines from pysen.exceptions import IncompatibleVersionError from pysen.path import change_dir +from pysen.py_version import VersionRepresentation from pysen.reporter import Reporter from pysen.setting import SettingBase, to_dash_case @@ -91,9 +92,10 @@ def export(self) -> Tuple[Sequence[str], Dict[str, Any]]: @functools.lru_cache(1) def _check_flake8_version() -> None: version = get_version("flake8") - if version.major != 3 or version.minor < 7: + minimum_supported = VersionRepresentation(3, 7) + if version < minimum_supported: raise IncompatibleVersionError( - "pysen only supports flake8 version >=3.7, <4. " + f"pysen only supports flake8 >= {minimum_supported}, " f"version {version} is not supported." ) @@ -104,7 +106,7 @@ def run( setting_path: pathlib.Path, sources: Iterable[pathlib.Path], ) -> int: - check_command_installed("flake8", "--version") + check_command_installed(*process_utils.add_python_executable("flake8", "--version")) _check_flake8_version() targets = [str(d) for d in sources] if len(targets) == 0: @@ -112,7 +114,9 @@ def run( cmd = ["flake8", "--config", str(setting_path)] + targets with change_dir(base_dir): - ret, stdout, _ = process_utils.run(cmd, reporter) + ret, stdout, _ = process_utils.run( + process_utils.add_python_executable(*cmd), reporter + ) diagnostics = parse_error_lines(stdout, logger=reporter.logger) reporter.report_diagnostics(list(diagnostics)) diff --git a/pysen/ext/isort_wrapper.py b/pysen/ext/isort_wrapper.py index 4b32410..0495986 100644 --- a/pysen/ext/isort_wrapper.py +++ b/pysen/ext/isort_wrapper.py @@ -121,7 +121,7 @@ def run( sources: Iterable[pathlib.Path], inplace_edit: bool, ) -> int: - check_command_installed("isort", "--version") + check_command_installed(*process_utils.add_python_executable("isort", "--version")) version = _get_isort_version() targets = [str(d) for d in sources] @@ -136,7 +136,9 @@ def run( cmd += targets with change_dir(base_dir): - ret, stdout, _ = process_utils.run(cmd, reporter) + ret, stdout, _ = process_utils.run( + process_utils.add_python_executable(*cmd), reporter + ) diagnostics = parse_error_diffs(stdout, _parse_file_path, logger=reporter.logger) reporter.report_diagnostics(list(diagnostics)) diff --git a/pysen/ext/mypy_wrapper.py b/pysen/ext/mypy_wrapper.py index 9c44cc0..e62d2e1 100644 --- a/pysen/ext/mypy_wrapper.py +++ b/pysen/ext/mypy_wrapper.py @@ -80,7 +80,7 @@ class MypySetting(SettingBase): _pysen_convert_abspath: bool = False @staticmethod - def very_strict(**kwargs: Any) -> "MypySetting": + def _top(**kwargs: Any) -> "MypySetting": updates = { "check_untyped_defs": True, "disallow_any_decorated": True, @@ -107,7 +107,7 @@ def very_strict(**kwargs: Any) -> "MypySetting": return MypySetting(**updates) # type: ignore @staticmethod - def strict(**kwargs: Any) -> "MypySetting": + def very_strict(**kwargs: Any) -> "MypySetting": updates = { "disallow_any_decorated": False, "disallow_any_unimported": False, @@ -115,6 +115,16 @@ def strict(**kwargs: Any) -> "MypySetting": "ignore_missing_imports": True, } updates.update(kwargs) + setting = MypySetting._top(**updates) + return setting + + @staticmethod + def strict(**kwargs: Any) -> "MypySetting": + updates = { + "warn_unused_ignores": False, + "disallow_any_generics": False, + } + updates.update(kwargs) setting = MypySetting.very_strict(**updates) return setting @@ -174,6 +184,7 @@ def export( @dataclasses.dataclass class MypyTarget: paths: List[pathlib.Path] + namespace_packages: bool = False @functools.lru_cache(1) @@ -193,7 +204,7 @@ def run( target: MypyTarget, require_diagnostics: bool, ) -> int: - check_command_installed("mypy", "--version") + check_command_installed(*process_utils.add_python_executable("mypy", "--version")) _check_mypy_version() target_paths = [str(resolve_path(base_dir, x)) for x in target.paths] @@ -212,9 +223,14 @@ def run( "--pretty", ] + if target.namespace_packages: + extra_options.append("--namespace-packages") + cmd = ["mypy"] + extra_options + ["--config-file", str(setting_path)] + target_paths with change_dir(base_dir): - ret, stdout, _ = process_utils.run(cmd, reporter) + ret, stdout, _ = process_utils.run( + process_utils.add_python_executable(*cmd), reporter + ) if require_diagnostics: diagnostics = parse_error_lines(stdout, logger=reporter.logger) diff --git a/pysen/process_utils.py b/pysen/process_utils.py index 7b79fed..2d08623 100644 --- a/pysen/process_utils.py +++ b/pysen/process_utils.py @@ -1,28 +1,40 @@ import contextlib import logging import subprocess +import sys from concurrent.futures import ThreadPoolExecutor -from typing import IO, List, Sequence, Tuple +from typing import IO, List, Optional, Sequence, Tuple from .reporter import Reporter -def _read_stream(stream: IO[bytes], reporter: Reporter, loglevel: int) -> str: +def _read_stream(stream: IO[str], reporter: Reporter, loglevel: int) -> str: ret: List[str] = [] - for s in stream: - line = s.decode("utf-8") + for line in stream: ret.append(line) reporter.process_output.log(loglevel, line.rstrip("\n")) return "".join(ret) +def add_python_executable(*cmd: str) -> Sequence[str]: + return [sys.executable, "-m"] + list(cmd) + + def run( cmd: Sequence[str], reporter: Reporter, stdout_loglevel: int = logging.INFO, stderr_loglevel: int = logging.WARNING, + encoding: Optional[str] = None, ) -> Tuple[int, str, str]: + # NOTE: As pysen doesn't configure `sys.stdout` with `errors=ignore` option, + # it may cause an error when unsupported characters in an environment are + # going to be printed. + # As such, `run` method returns strings of printable characters in the environment + # so that pysen doesn't need to reconfigure `sys.stdout`. + encoding = encoding or sys.stdout.encoding + returncode: int = -1 stdout: str = "" stderr: str = "" @@ -30,7 +42,14 @@ def run( with contextlib.ExitStack() as stack: reporter.report_command(" ".join(cmd)) proc = stack.enter_context( - subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding=encoding, + universal_newlines=True, + errors="ignore", + ) ) try: pool = stack.enter_context(ThreadPoolExecutor(max_workers=2)) diff --git a/pysen/py_version.py b/pysen/py_version.py index f807bda..e2ae2a0 100644 --- a/pysen/py_version.py +++ b/pysen/py_version.py @@ -72,6 +72,17 @@ def __eq__(self, other: object) -> bool: def is_compatible(self, other: "VersionRepresentation") -> bool: return self.major == other.major and self.minor == other.minor + def __lt__(self, other: "VersionRepresentation") -> bool: + """Compare with a given VersionRepresentation + + .. warning:: + This method does NOT consider pre_release versions. + """ + # check whether self < other + lhs = (self.major, self.minor, self.patch or -1) + rhs = (other.major, other.minor, other.patch or -1) + return lhs < rhs + @dataclasses.dataclass(frozen=True) class PythonVersion(VersionRepresentation): diff --git a/pysen/pyproject.py b/pysen/pyproject.py index 67c1f20..2ff6cec 100644 --- a/pysen/pyproject.py +++ b/pysen/pyproject.py @@ -15,6 +15,8 @@ _logger = logging.getLogger(__name__) TConfig = TypeVar("TConfig") +# NOTE: The following names are in config resolution order +CONFIG_FILE_NAMES = ["pysen.toml", "pyproject.toml"] def resolve_inheritance( @@ -105,20 +107,33 @@ def load_manifest(path: pathlib.Path) -> ManifestBase: return build(components, path, external_builder) -def _find_recursive() -> Optional[pathlib.Path]: - current = pathlib.Path.cwd().resolve() +def _check_section_exists(config_path: pathlib.Path) -> bool: + pyproject = tomlkit.loads(config_path.read_text()) + return has_tool_section("jiro", pyproject) or has_tool_section("pysen", pyproject) - while True: - path = current / "pyproject.toml" + +def find_config(target_dir: pathlib.Path) -> Optional[pathlib.Path]: + # NOTE: This method doesn't ensure if a config is valid for the configuration model + for name in CONFIG_FILE_NAMES: + path = target_dir / name if path.exists() and path.is_file(): - pyproject = tomlkit.loads(path.read_text()) - if has_tool_section("jiro", pyproject) or has_tool_section( - "pysen", pyproject - ): - _logger.debug(f"successfully found pyproject.toml: {path}") + if _check_section_exists(path): + _logger.debug(f"successfully found config file: {path}") return path - _logger.debug(f"found a file, but pysen.tool doesn't exist: {path}") + _logger.debug( + f"found a file, but [tool.pysen] section doesn't exist: {path}" + ) + + return None + + +def find_config_recursive(base_dir: pathlib.Path) -> Optional[pathlib.Path]: + current = base_dir.resolve() + while True: + config = find_config(current) + if config is not None: + return config # reached root if current.parent == current: @@ -137,11 +152,12 @@ def find_pyproject(path: Optional[pathlib.Path] = None) -> pathlib.Path: return path else: - p = _find_recursive() + cwd = pathlib.Path.cwd() + p = find_config_recursive(cwd) if p is None: raise FileNotFoundError( - "Could not find a pyproject.toml file " - "containing a [tool.pysen] section " + "Could not find either a pyproject.toml file or " + "a pysen.toml file containing a [tool.pysen] section " "in this or any of its parent directories. \n" "The `--loglevel debug option` may help." ) diff --git a/pysen/pyproject_model.py b/pysen/pyproject_model.py index 814c7f0..0772480 100644 --- a/pysen/pyproject_model.py +++ b/pysen/pyproject_model.py @@ -335,7 +335,7 @@ def _load_pysen_section(path: pathlib.Path) -> Dict[str, Any]: section = pyproject["tool"]["pysen"] elif has_tool_section("jiro", pyproject): _logger.warning( - "jiro section under pyproject.toml is deprecated. Use pysen instead." + "jiro section under a config file is deprecated. Use pysen instead." ) section = pyproject["tool"]["jiro"] else: @@ -396,13 +396,13 @@ def _check_version( ) -> None: if config_version is None: _logger.warning( - "Consider specifying 'version' under [tool.pysen] section in your pyproject.toml " + "Consider specifying 'version' under [tool.pysen] section in your config " "to check compliance against the version of the installed pysen. " f"(File: {file_path})" ) elif not config_version.is_compatible(actual_version): _logger.warning( - f"pyproject.toml specifies version {config_version}, " + f"config specifies version {config_version}, " f"but the pysen you are using is version {actual_version}, " "which might not be compatible. " f"(File: {file_path})" diff --git a/setup.py b/setup.py index f6e390e..b59eb17 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ "lint": [ "black>=19.10b0,<=20.8", "flake8-bugbear", # flake8 doesn't have a dependency for bugbear plugin - "flake8>=3.7,<4", + "flake8>=3.7,<5", "isort>=4.3,<5.2.0", "mypy>=0.770,<0.800", ], diff --git a/tests/test_mypy.py b/tests/test_mypy.py index 9158a3d..d46b685 100644 --- a/tests/test_mypy.py +++ b/tests/test_mypy.py @@ -7,6 +7,7 @@ from pysen import mypy from pysen.mypy import _get_differences_from_base +from pysen.process_utils import add_python_executable from pysen.reporter import Reporter from pysen.runner_options import PathContext, RunOptions from pysen.setting import SettingFile @@ -88,7 +89,7 @@ def test_commands(reporter: Reporter) -> None: m = mypy.Mypy( mypy_targets=[mypy.MypyTarget([pathlib.Path("/bar"), pathlib.Path("baz")])] ) - expected_cmds = [ + expected_cmds = add_python_executable( "mypy", "--show-absolute-path", "--no-color-output", @@ -98,7 +99,7 @@ def test_commands(reporter: Reporter) -> None: "/setting/setup.cfg", "/bar", "/foo/baz", - ] + ) cmd = m.create_command( "lint", PathContext(pathlib.Path("/foo"), pathlib.Path("/setting")), diff --git a/tests/test_process_utils.py b/tests/test_process_utils.py index fcd8fe0..fca17f4 100644 --- a/tests/test_process_utils.py +++ b/tests/test_process_utils.py @@ -2,6 +2,7 @@ import logging import os import pathlib +import sys import tempfile from typing import List @@ -37,6 +38,8 @@ """ +TEST_UNICODE = b"pysen\xe3\x81\xb1\xe3\x81\x84\xe3\x81\x9b\xe3\x82\x9312345" + class FakeHandler(logging.Handler): def __init__(self) -> None: @@ -80,18 +83,42 @@ def test__read_stream(sample_str: str) -> None: reporter.process_output.handlers.clear() reporter.process_output.addHandler(handler) - ret = _read_stream(io.BytesIO(temp_path.read_bytes()), reporter, logging.INFO) + ret = _read_stream(io.StringIO(temp_path.read_text()), reporter, logging.INFO) expected = sample_str assert ret == expected assert handler.messages == expected.splitlines() handler.messages.clear() - ret = _read_stream(io.BytesIO(temp_path.read_bytes()), reporter, logging.DEBUG) + ret = _read_stream(io.StringIO(temp_path.read_text()), reporter, logging.DEBUG) assert ret == expected assert handler.messages == [] +def test_run_encoding() -> None: + with tempfile.TemporaryDirectory() as td: + temp_dir = pathlib.Path(td) + temp_path = temp_dir / "file" + temp_path.write_bytes(TEST_UNICODE) + reporter = Reporter("foo") + + expected_stdout = temp_path.read_text(sys.stdout.encoding, errors="ignore") + expected_utf8 = temp_path.read_text("utf-8", errors="ignore") + expected_ascii = temp_path.read_text("ascii", errors="ignore") + + ret, stdout, stderr = run(["cat", str(temp_path)], reporter) + assert ret == 0 + assert stdout == expected_stdout + + ret, stdout, _ = run(["cat", str(temp_path)], reporter, encoding="utf-8") + assert ret == 0 + assert stdout == expected_utf8 + + ret, stdout, _ = run(["cat", str(temp_path)], reporter, encoding="ascii") + assert ret == 0 + assert stdout == expected_ascii == "pysen12345" + + def test_run() -> None: assert os.getenv("LANG", "C") == "C", "Did you run pytest through tox?" with tempfile.TemporaryDirectory() as td: diff --git a/tests/test_py_version.py b/tests/test_py_version.py index 40788a4..9a673ab 100644 --- a/tests/test_py_version.py +++ b/tests/test_py_version.py @@ -30,6 +30,26 @@ def test_version_ops() -> None: assert VersionRepresentation(3, 6) == "3" +def test_version_comp() -> None: + def assert_rhs_is_larger( + lhs: VersionRepresentation, rhs: VersionRepresentation + ) -> None: + assert lhs < rhs + assert not rhs < lhs + + assert_rhs_is_larger(VersionRepresentation(0, 4, 5), VersionRepresentation(1, 2, 3)) + assert_rhs_is_larger(VersionRepresentation(1, 0), VersionRepresentation(2, 0)) + assert_rhs_is_larger(VersionRepresentation(0, 5), VersionRepresentation(0, 6)) + assert_rhs_is_larger(VersionRepresentation(0, 5), VersionRepresentation(0, 5, 1)) + + assert not VersionRepresentation(0, 5, 1, "b0") < VersionRepresentation( + 0, 5, 1, "b1" + ) + assert not VersionRepresentation(0, 5, 1, "b1") < VersionRepresentation( + 0, 5, 1, "b0" + ) + + def test_is_compatible() -> None: assert VersionRepresentation(0, 5).is_compatible(VersionRepresentation(0, 5)) assert VersionRepresentation(0, 5).is_compatible(VersionRepresentation(0, 5, 7)) diff --git a/tests/test_pyproject.py b/tests/test_pyproject.py index 51b9fe6..5277513 100644 --- a/tests/test_pyproject.py +++ b/tests/test_pyproject.py @@ -1,6 +1,7 @@ import dataclasses import pathlib -from typing import Callable, Optional +import tempfile +from typing import Callable, Iterator, Optional import dacite import pytest @@ -28,6 +29,12 @@ assert EXAMPLE_PYPROJECT_PATH.exists() +@pytest.fixture +def temp_dir() -> Iterator[pathlib.Path]: + with tempfile.TemporaryDirectory() as td: + yield pathlib.Path(td) + + def test_find_pyproject() -> None: with change_dir(BASE_DIR): assert pyproject.find_pyproject() == PYPROJECT_PATH @@ -220,3 +227,76 @@ def updater(lhs: _Model, rhs: _Model) -> _Model: assert f"Circular dependency detected. {base3} was visited more than once." in str( e ) + + +def test__check_section_exists(temp_dir: pathlib.Path) -> None: + test_file = temp_dir / "foo.toml" + test_file.write_text("x = 0") + assert not pyproject._check_section_exists(test_file) + + test_file.write_text( + """[tool] +x = 0""" + ) + assert not pyproject._check_section_exists(test_file) + + test_file.write_text( + """[tool.pysen] +x = 0""" + ) + assert pyproject._check_section_exists(test_file) + + test_file.write_text( + """[tool.pysen2] +x = 0""" + ) + assert not pyproject._check_section_exists(test_file) + + test_file.write_text( + """[tool.pysen] +x = 0 +[tool.foo] +y = 1""" + ) + assert pyproject._check_section_exists(test_file) + + +def test_find_config(temp_dir: pathlib.Path) -> None: + pysen_config = """[tool.pysen] +x = 0""" + other_config = """[tool.foo] +x = 0""" + + pyproject_file = temp_dir / "pyproject.toml" + pysen_file = temp_dir / "pysen.toml" + other_file = temp_dir / "foo.toml" + + other_file.write_text(pysen_config) + assert pyproject.find_config(temp_dir) is None + + pyproject_file.write_text(other_config) + assert pyproject.find_config(temp_dir) is None + + pyproject_file.write_text(pysen_config) + assert pyproject.find_config(temp_dir) == pyproject_file + + # checks that the config resolution order is pysen_file -> pyproject_file + pysen_file.write_text(pysen_config) + assert pyproject.find_config(temp_dir) == pysen_file + + pysen_file.write_text(other_config) + assert pyproject.find_config(temp_dir) == pyproject_file + + +def test_find_config_recursive(temp_dir: pathlib.Path) -> None: + assert pyproject.find_config_recursive(BASE_DIR) == PYPROJECT_PATH + assert pyproject.find_config_recursive(BASE_DIR / "fakes/configs") == PYPROJECT_PATH + assert pyproject.find_config_recursive(EXAMPLE_DIR) == EXAMPLE_PYPROJECT_PATH + assert pyproject.find_config_recursive(temp_dir) is None + + temp_pyproject = temp_dir / "pyproject.toml" + temp_pyproject.write_text( + """[tool.pysen] +x = 1""" + ) + assert pyproject.find_config_recursive(temp_dir) == temp_pyproject diff --git a/tests/test_pyproject_model.py b/tests/test_pyproject_model.py index 2a4475b..f080652 100644 --- a/tests/test_pyproject_model.py +++ b/tests/test_pyproject_model.py @@ -161,6 +161,10 @@ def test__parse_mypy_target() -> None: target = _parse_mypy_target(base_dir, {"paths": ["a", "b", "/d"]}) assert target.paths == [base_dir / "a", base_dir / "b", pathlib.Path("/d")] + target = _parse_mypy_target(base_dir, {"paths": ["a"], "namespace_packages": True}) + assert target.paths == [base_dir / "a"] + assert target.namespace_packages + def test__parse_mypy_targets() -> None: base_dir = pathlib.Path("/foo") diff --git a/tox.ini b/tox.ini index 0ba95ec..5bf4e11 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{36,37}-dacite{110,120,150}-isort43-black20, py37-dacite150-isort{43,50,51}-black{19,20}, py{38,39}-dacite150-isort51-black20, development +envlist = py{36,37}-dacite{110,120,150}-isort43-black20, py37-dacite150-isort{43,50,51}-black{19,20}, py{38,39}-dacite150-isort51-black20, development, latest [testenv] deps = @@ -12,8 +12,8 @@ deps = isort51: isort>=5.1.0,<5.2.0 black19: black==19.10b0 black20: black==20.8b1 - flake8==3.8.3 - flake8-bugbear==20.1.4 + flake8==4.0.1 + flake8-bugbear==21.9.2 mypy==0.782 extras = lint @@ -23,6 +23,15 @@ commands = setenv = LANG = C +[testenv:latest] +basepython = python3.7 +deps = + pipenv==2020.11.15 +commands = + pipenv sync + pipenv run pip install -U black + pipenv run pytest + [testenv:development] basepython = python3.7 deps =