From 20cdda5e8bbd9f83d64b154f6b4fcd28216c63e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 03:09:00 +0000 Subject: [PATCH 01/37] chore(deps): bump pypa/gh-action-pypi-publish from 1.4.2 to 1.5.0 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.4.2 to 1.5.0. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.4.2...v1.5.0) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e2d682dd..2f5a995dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,7 +110,7 @@ jobs: name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@v1.4.2 + - uses: pypa/gh-action-pypi-publish@v1.5.0 with: user: __token__ password: ${{ secrets.pypi_password }} From 34fa7e4774f5fdeaf3aaf26efc8b8a9e62c63f6c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 22 Jan 2022 01:49:01 -0500 Subject: [PATCH 02/37] tests: fix config options (#576) --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cf3999259..e75233134 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,9 +23,11 @@ ignore_missing_imports = true [tool.pytest.ini_options] -min_version = "6.0" -addopts = "-v -ra --cov-config=setup.cfg" +minversion = "6.0" +addopts = ["-ra", "--cov-config=setup.cfg", "--strict-markers", "--strict-config"] norecursedirs = ["examples", "experiments"] +# filterwarnings = ["error"] +log_cli_level = "info" required_plugins = ["pytest-timeout", "pytest-mock"] timeout = 300 optional_tests = """ From bd29e7f21b1a9856d81553f149c68d07fbc38ce9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 24 Jan 2022 18:20:49 -0500 Subject: [PATCH 03/37] feat: support forcing color/nocolor (#575) * feat: support forcing color/nocolor * tests: ignore occasional PyPy error * chore: remove Python 3 __div__ workaround * fix: guard style --- CHANGELOG.rst | 8 ++++++++ plumbum/colorlib/styles.py | 16 ++++++++++------ plumbum/machines/paramiko_machine.py | 6 ++---- plumbum/path/base.py | 4 +--- tests/test_remote.py | 1 + 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af23eaeff..99ddc27db 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,11 @@ +1.8.0 +----- + +* Drop Python 2.7 and 3.5 support (`#573 `_) +* Color: support ``NO_COLOR``/``FORCE_COLOR`` (`#575 `_) + + + 1.7.2 ----- diff --git a/plumbum/colorlib/styles.py b/plumbum/colorlib/styles.py index 9607d59d4..e007944af 100644 --- a/plumbum/colorlib/styles.py +++ b/plumbum/colorlib/styles.py @@ -39,22 +39,26 @@ def get_color_repr(): """Gets best colors for current system.""" + if "NO_COLOR" in os.environ: + return 0 + if os.environ.get("FORCE_COLOR", "0") in {"0", "1", "2", "3", "4"}: + return int(os.environ["FORCE_COLOR"]) if not sys.stdout.isatty(): - return False + return 0 term = os.environ.get("TERM", "") # Some terminals set TERM=xterm for compatibility if term.endswith("256color") or term == "xterm": return 3 if platform.system() == "Darwin" else 4 - elif term.endswith("16color"): + if term.endswith("16color"): return 2 - elif term == "screen": + if term == "screen": return 1 - elif os.name == "nt": + if os.name == "nt": return 0 - else: - return 3 + + return 3 class ColorNotFound(Exception): diff --git a/plumbum/machines/paramiko_machine.py b/plumbum/machines/paramiko_machine.py index 3c8eb6a55..e198f0346 100644 --- a/plumbum/machines/paramiko_machine.py +++ b/plumbum/machines/paramiko_machine.py @@ -18,11 +18,9 @@ except ImportError: class paramiko: # type: ignore - def __nonzero__(self): + def __bool__(self): return False - __bool__ = __nonzero__ - def __getattr__(self, name): raise ImportError("No module named paramiko") @@ -433,7 +431,7 @@ def _path_stat(self, fn): except OSError as e: if e.errno == errno.ENOENT: return None - raise OSError(e.errno) + raise res = StatRes( ( st.st_mode, diff --git a/plumbum/path/base.py b/plumbum/path/base.py index 301e6ccca..7b86c68b9 100644 --- a/plumbum/path/base.py +++ b/plumbum/path/base.py @@ -31,12 +31,10 @@ class Path(str, ABC): def __repr__(self): return f"<{self.__class__.__name__} {str(self)}>" - def __div__(self, other): + def __truediv__(self, other): """Joins two paths""" return self.join(other) - __truediv__ = __div__ - def __getitem__(self, key): if type(key) == str or isinstance(key, Path): return self / key diff --git a/tests/test_remote.py b/tests/test_remote.py index 568fe5597..7e94aa53c 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -356,6 +356,7 @@ def test_session(self): _, out, _ = sh.run("ls -a") assert ".bashrc" in out or ".bash_profile" in out + @pytest.mark.xfail(env.PYPY, reason="PyPy sometimes fails here", strict=False) def test_env(self): with self._connect() as rem: with pytest.raises(ProcessExecutionError): From e6fe1a95ee7e18268941296494cc89713fc8e5d3 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 27 Jan 2022 14:34:23 -0500 Subject: [PATCH 04/37] chore: remove setdoc (not needed in Py3) (#577) * chore: remove setdoc (not needed in Py3) * chore: minor Python 2 cleanup --- docs/cli.rst | 3 +- plumbum/cli/config.py | 5 --- plumbum/lib.py | 11 ------ plumbum/machines/paramiko_machine.py | 5 --- plumbum/machines/remote.py | 8 +---- plumbum/machines/ssh_machine.py | 7 +--- plumbum/path/base.py | 8 ++--- plumbum/path/local.py | 52 +++++----------------------- plumbum/path/remote.py | 52 +++++----------------------- 9 files changed, 23 insertions(+), 128 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 8b113e4d0..7b45ed328 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -420,8 +420,7 @@ pass the validators in the decorator matching the names in the main function. Fo def main(self, infile, *outfiles): "infile is a path, outfiles are a list of paths, proper errors are given" -If you only want to run your application in Python 3, you can also use annotations to -specify the validators. For example:: +You can also use annotations to specify the validators. For example:: class MyApp(cli.Application): def main(self, infile : cli.ExistingFile, *outfiles : cli.NonexistentPath): diff --git a/plumbum/cli/config.py b/plumbum/cli/config.py index c01e7efdb..c128474ab 100644 --- a/plumbum/cli/config.py +++ b/plumbum/cli/config.py @@ -2,7 +2,6 @@ from configparser import ConfigParser, NoOptionError, NoSectionError from plumbum import local -from plumbum.lib import _setdoc class ConfigBase(ABC): @@ -85,12 +84,10 @@ def __init__(self, filename): super().__init__(filename) self.parser = ConfigParser() - @_setdoc(ConfigBase) def read(self): self.parser.read(self.filename) super().read() - @_setdoc(ConfigBase) def write(self): with open(self.filename, "w") as f: self.parser.write(f) @@ -104,7 +101,6 @@ def _sec_opt(cls, option): sec, option = option.split(".", 1) return sec, option - @_setdoc(ConfigBase) def _get(self, option): sec, option = self._sec_opt(option) @@ -113,7 +109,6 @@ def _get(self, option): except (NoSectionError, NoOptionError): raise KeyError(f"{sec}:{option}") - @_setdoc(ConfigBase) def _set(self, option, value): sec, option = self._sec_opt(option) try: diff --git a/plumbum/lib.py b/plumbum/lib.py index e6b60f63c..0a2f7b174 100644 --- a/plumbum/lib.py +++ b/plumbum/lib.py @@ -7,17 +7,6 @@ IS_WIN32 = sys.platform == "win32" -def _setdoc(super): # @ReservedAssignment - """This inherits the docs on the current class. Not really needed for Python 3.5, - due to new behavior of inspect.getdoc, but still doesn't hurt.""" - - def deco(func): - func.__doc__ = getattr(getattr(super, func.__name__, None), "__doc__", None) - return func - - return deco - - class ProcInfo: def __init__(self, pid, uid, stat, args): self.pid = pid diff --git a/plumbum/machines/paramiko_machine.py b/plumbum/machines/paramiko_machine.py index e198f0346..9cd7c6a2b 100644 --- a/plumbum/machines/paramiko_machine.py +++ b/plumbum/machines/paramiko_machine.py @@ -5,7 +5,6 @@ from plumbum.commands.base import shquote from plumbum.commands.processes import ProcessLineTimedOut, iter_lines -from plumbum.lib import _setdoc from plumbum.machines.base import PopenAddons from plumbum.machines.remote import BaseRemoteMachine from plumbum.machines.session import ShellSession @@ -286,7 +285,6 @@ def sftp(self): self._sftp = self._client.open_sftp() return self._sftp - @_setdoc(BaseRemoteMachine) def session( self, isatty=False, term="vt100", width=80, height=24, new_session=False ): @@ -304,7 +302,6 @@ def session( proc = ParamikoPopen([""], stdin, stdout, stderr, self.custom_encoding) return ShellSession(proc, self.custom_encoding, isatty) - @_setdoc(BaseRemoteMachine) def popen( self, args, @@ -339,7 +336,6 @@ def popen( stderr_file=stderr, ) - @_setdoc(BaseRemoteMachine) def download(self, src, dst): if isinstance(src, LocalPath): raise TypeError(f"src of download cannot be {src!r}") @@ -363,7 +359,6 @@ def _download(self, src, dst): else: self.sftp.get(str(src), str(dst)) - @_setdoc(BaseRemoteMachine) def upload(self, src, dst): if isinstance(src, RemotePath): raise TypeError(f"src of upload cannot be {src!r}") diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index 0d3be092b..d69a9c753 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -3,7 +3,7 @@ from tempfile import NamedTemporaryFile from plumbum.commands import CommandNotFound, ConcreteCommand, shquote -from plumbum.lib import ProcInfo, _setdoc +from plumbum.lib import ProcInfo from plumbum.machines.base import BaseMachine from plumbum.machines.env import BaseEnv from plumbum.machines.local import LocalPath @@ -39,22 +39,18 @@ def __init__(self, remote): self._orig = self._curr.copy() BaseEnv.__init__(self, self.remote.path, ":") - @_setdoc(BaseEnv) def __delitem__(self, name): BaseEnv.__delitem__(self, name) self.remote._session.run(f"unset {name}") - @_setdoc(BaseEnv) def __setitem__(self, name, value): BaseEnv.__setitem__(self, name, value) self.remote._session.run(f"export {name}={shquote(value)}") - @_setdoc(BaseEnv) def pop(self, name, *default): BaseEnv.pop(self, name, *default) self.remote._session.run(f"unset {name}") - @_setdoc(BaseEnv) def update(self, *args, **kwargs): BaseEnv.update(self, *args, **kwargs) self.remote._session.run( @@ -435,11 +431,9 @@ def _path_link(self, src, dst, symlink): "ln {} {} {}".format("-s" if symlink else "", shquote(src), shquote(dst)) ) - @_setdoc(BaseEnv) def expand(self, expr): return self._session.run(f"echo {expr}")[1].strip() - @_setdoc(BaseEnv) def expanduser(self, expr): if not any(part.startswith("~") for part in expr.split("/")): return expr diff --git a/plumbum/machines/ssh_machine.py b/plumbum/machines/ssh_machine.py index dbae08151..be1f3dc83 100644 --- a/plumbum/machines/ssh_machine.py +++ b/plumbum/machines/ssh_machine.py @@ -1,7 +1,7 @@ import warnings from plumbum.commands import ProcessExecutionError, shquote -from plumbum.lib import IS_WIN32, _setdoc +from plumbum.lib import IS_WIN32 from plumbum.machines.local import local from plumbum.machines.remote import BaseRemoteMachine from plumbum.machines.session import ShellSession @@ -131,7 +131,6 @@ def __init__( def __str__(self): return f"ssh://{self._fqhost}" - @_setdoc(BaseRemoteMachine) def popen(self, args, ssh_opts=(), env=None, cwd=None, **kwargs): cmdline = [] cmdline.extend(ssh_opts) @@ -203,7 +202,6 @@ def daemonic_popen(self, command, cwd=".", stdout=None, stderr=None, append=True proc.stdout.close() proc.stderr.close() - @_setdoc(BaseRemoteMachine) def session(self, isatty=False, new_session=False): return ShellSession( self.popen( @@ -299,7 +297,6 @@ def _translate_drive_letter(self, path): path = "/" + path.replace(":", "").replace("\\", "/") return path - @_setdoc(BaseRemoteMachine) def download(self, src, dst): if isinstance(src, LocalPath): raise TypeError(f"src of download cannot be {src!r}") @@ -312,7 +309,6 @@ def download(self, src, dst): dst = self._translate_drive_letter(dst) self._scp_command(f"{self._fqhost}:{shquote(src)}", dst) - @_setdoc(BaseRemoteMachine) def upload(self, src, dst): if isinstance(src, RemotePath): raise TypeError(f"src of upload cannot be {src!r}") @@ -382,7 +378,6 @@ def _translate_drive_letter(self, path): # pscp takes care of windows paths automatically return path - @_setdoc(BaseRemoteMachine) def session(self, isatty=False, new_session=False): return ShellSession( self.popen((), (["-t"] if isatty else ["-T"]), new_session=new_session), diff --git a/plumbum/path/base.py b/plumbum/path/base.py index 7b86c68b9..d45906b5e 100644 --- a/plumbum/path/base.py +++ b/plumbum/path/base.py @@ -80,11 +80,9 @@ def __hash__(self): else: return hash(str(self).lower()) - def __nonzero__(self): + def __bool__(self): return bool(str(self)) - __bool__ = __nonzero__ - def __fspath__(self): """Added for Python 3.6 support""" return str(self) @@ -478,11 +476,9 @@ def __le__(self, other): def __hash__(self): return hash(str(self)) - def __nonzero__(self): + def __bool__(self): return bool(str(self)) - __bool__ = __nonzero__ - def up(self, count=1): return RelativePath(self.parts[:-count]) diff --git a/plumbum/path/local.py b/plumbum/path/local.py index 58c1a2232..b4080fe2a 100644 --- a/plumbum/path/local.py +++ b/plumbum/path/local.py @@ -6,7 +6,7 @@ import sys from contextlib import contextmanager -from plumbum.lib import IS_WIN32, _setdoc +from plumbum.lib import IS_WIN32 from plumbum.path.base import FSUser, Path from plumbum.path.remote import RemotePath @@ -68,18 +68,15 @@ def _get_info(self): def _form(self, *parts): return LocalPath(*parts) - @property # type: ignore - @_setdoc(Path) + @property def name(self): return os.path.basename(str(self)) - @property # type: ignore - @_setdoc(Path) + @property def dirname(self): return LocalPath(os.path.dirname(str(self))) - @property # type: ignore - @_setdoc(Path) + @property def suffix(self): return os.path.splitext(str(self))[1] @@ -94,65 +91,52 @@ def suffixes(self): else: return list(reversed(exts)) - @property # type: ignore - @_setdoc(Path) + @property def uid(self): uid = self.stat().st_uid name = getpwuid(uid)[0] return FSUser(uid, name) - @property # type: ignore - @_setdoc(Path) + @property def gid(self): gid = self.stat().st_gid name = getgrgid(gid)[0] return FSUser(gid, name) - @_setdoc(Path) def join(self, *others): return LocalPath(self, *others) - @_setdoc(Path) def list(self): return [self / fn for fn in os.listdir(str(self))] - @_setdoc(Path) def iterdir(self): try: return (self / fn.name for fn in os.scandir(str(self))) except AttributeError: return (self / fn for fn in os.listdir(str(self))) - @_setdoc(Path) def is_dir(self): return os.path.isdir(str(self)) - @_setdoc(Path) def is_file(self): return os.path.isfile(str(self)) - @_setdoc(Path) def is_symlink(self): return os.path.islink(str(self)) - @_setdoc(Path) def exists(self): return os.path.exists(str(self)) - @_setdoc(Path) def stat(self): return os.stat(str(self)) - @_setdoc(Path) def with_name(self, name): return LocalPath(self.dirname) / name - @property # type: ignore - @_setdoc(Path) + @property def stem(self): return self.name.rsplit(os.path.extsep)[0] - @_setdoc(Path) def with_suffix(self, suffix, depth=1): if suffix and not suffix.startswith(os.path.extsep) or suffix == os.path.extsep: raise ValueError("Invalid suffix %r" % (suffix)) @@ -162,14 +146,12 @@ def with_suffix(self, suffix, depth=1): name, _ = os.path.splitext(name) return LocalPath(self.dirname) / (name + suffix) - @_setdoc(Path) def glob(self, pattern): fn = lambda pat: [ LocalPath(m) for m in glob.glob(os.path.join(glob.escape(str(self)), pat)) ] return self._glob(pattern, fn) - @_setdoc(Path) def delete(self): if not self.exists(): return @@ -184,14 +166,12 @@ def delete(self): if ex.errno != errno.ENOENT: raise - @_setdoc(Path) def move(self, dst): if isinstance(dst, RemotePath): raise TypeError(f"Cannot move local path {self} to {dst!r}") shutil.move(str(self), str(dst)) return LocalPath(dst) - @_setdoc(Path) def copy(self, dst, override=None): if isinstance(dst, RemotePath): raise TypeError(f"Cannot copy local path {self} to {dst!r}") @@ -209,7 +189,6 @@ def copy(self, dst, override=None): shutil.copy2(str(self), str(dst)) return dst - @_setdoc(Path) def mkdir(self, mode=0o777, parents=True, exist_ok=True): if not self.exists() or not exist_ok: try: @@ -223,11 +202,9 @@ def mkdir(self, mode=0o777, parents=True, exist_ok=True): if ex.errno != errno.EEXIST or not exist_ok: raise - @_setdoc(Path) def open(self, mode="r"): return open(str(self), mode) - @_setdoc(Path) def read(self, encoding=None, mode="r"): if encoding and "b" not in mode: mode = mode + "b" @@ -237,7 +214,6 @@ def read(self, encoding=None, mode="r"): data = data.decode(encoding) return data - @_setdoc(Path) def write(self, data, encoding=None, mode=None): if encoding: data = data.encode(encoding) @@ -249,12 +225,10 @@ def write(self, data, encoding=None, mode=None): with self.open(mode) as f: f.write(data) - @_setdoc(Path) def touch(self): with open(str(self), "a"): os.utime(str(self), None) - @_setdoc(Path) def chown(self, owner=None, group=None, recursive=None): if not hasattr(os, "chown"): raise OSError("os.chown() not supported") @@ -273,17 +247,14 @@ def chown(self, owner=None, group=None, recursive=None): for subpath in self.walk(): os.chown(str(subpath), uid, gid) - @_setdoc(Path) def chmod(self, mode): if not hasattr(os, "chmod"): raise OSError("os.chmod() not supported") os.chmod(str(self), mode) - @_setdoc(Path) def access(self, mode=0): return os.access(str(self), self._access_mode_to_flags(mode)) - @_setdoc(Path) def link(self, dst): if isinstance(dst, RemotePath): raise TypeError( @@ -300,7 +271,6 @@ def link(self, dst): else: local["cmd"]("/C", "mklink", "/H", str(dst), str(self)) - @_setdoc(Path) def symlink(self, dst): if isinstance(dst, RemotePath): raise TypeError( @@ -317,7 +287,6 @@ def symlink(self, dst): else: local["cmd"]("/C", "mklink", str(dst), str(self)) - @_setdoc(Path) def unlink(self): try: if hasattr(os, "symlink") or not self.is_dir(): @@ -331,17 +300,14 @@ def unlink(self): if ex.errno != errno.ENOENT: raise - @_setdoc(Path) def as_uri(self, scheme="file"): return urlparse.urljoin(str(scheme) + ":", urllib.pathname2url(str(self))) - @property # type: ignore - @_setdoc(Path) + @property def drive(self): return os.path.splitdrive(str(self))[0] - @property # type: ignore - @_setdoc(Path) + @property def root(self): return os.path.sep diff --git a/plumbum/path/remote.py b/plumbum/path/remote.py index a6aa8ff3d..b9afba68a 100644 --- a/plumbum/path/remote.py +++ b/plumbum/path/remote.py @@ -5,7 +5,6 @@ from contextlib import contextmanager from plumbum.commands import ProcessExecutionError, shquote -from plumbum.lib import _setdoc from plumbum.path.base import FSUser, Path @@ -84,27 +83,23 @@ def _form(self, *parts): def _path(self): return str(self) - @property # type: ignore - @_setdoc(Path) + @property def name(self): if "/" not in str(self): return str(self) return str(self).rsplit("/", 1)[1] - @property # type: ignore - @_setdoc(Path) + @property def dirname(self): if "/" not in str(self): return str(self) return self.__class__(self.remote, str(self).rsplit("/", 1)[0]) - @property # type: ignore - @_setdoc(Path) + @property def suffix(self): return "." + self.name.rsplit(".", 1)[1] - @property # type: ignore - @_setdoc(Path) + @property def suffixes(self): name = self.name exts = [] @@ -113,14 +108,12 @@ def suffixes(self): exts.append("." + ext) return list(reversed(exts)) - @property # type: ignore - @_setdoc(Path) + @property def uid(self): uid, name = self.remote._path_getuid(self) return FSUser(int(uid), name) - @property # type: ignore - @_setdoc(Path) + @property def gid(self): gid, name = self.remote._path_getgid(self) return FSUser(int(gid), name) @@ -128,59 +121,49 @@ def gid(self): def _get_info(self): return (self.remote, self._path) - @_setdoc(Path) def join(self, *parts): return RemotePath(self.remote, self, *parts) - @_setdoc(Path) def list(self): if not self.is_dir(): return [] return [self.join(fn) for fn in self.remote._path_listdir(self)] - @_setdoc(Path) def iterdir(self): if not self.is_dir(): return () return (self.join(fn) for fn in self.remote._path_listdir(self)) - @_setdoc(Path) def is_dir(self): res = self.remote._path_stat(self) if not res: return False return res.text_mode == "directory" - @_setdoc(Path) def is_file(self): res = self.remote._path_stat(self) if not res: return False return res.text_mode in ("regular file", "regular empty file") - @_setdoc(Path) def is_symlink(self): res = self.remote._path_stat(self) if not res: return False return res.text_mode == "symbolic link" - @_setdoc(Path) def exists(self): return self.remote._path_stat(self) is not None - @_setdoc(Path) def stat(self): res = self.remote._path_stat(self) if res is None: raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), "") return res - @_setdoc(Path) def with_name(self, name): return self.__class__(self.remote, self.dirname) / name - @_setdoc(Path) def with_suffix(self, suffix, depth=1): if suffix and not suffix.startswith(".") or suffix == ".": raise ValueError("Invalid suffix %r" % (suffix)) @@ -190,14 +173,12 @@ def with_suffix(self, suffix, depth=1): name, _ = name.rsplit(".", 1) return self.__class__(self.remote, self.dirname) / (name + suffix) - @_setdoc(Path) def glob(self, pattern): fn = lambda pat: [ RemotePath(self.remote, m) for m in self.remote._path_glob(self, pat) ] return self._glob(pattern, fn) - @_setdoc(Path) def delete(self): if not self.exists(): return @@ -205,7 +186,6 @@ def delete(self): unlink = delete - @_setdoc(Path) def move(self, dst): if isinstance(dst, RemotePath): if dst.remote is not self.remote: @@ -217,7 +197,6 @@ def move(self, dst): ) self.remote._path_move(self, dst) - @_setdoc(Path) def copy(self, dst, override=False): if isinstance(dst, RemotePath): if dst.remote is not self.remote: @@ -239,7 +218,6 @@ def copy(self, dst, override=False): self.remote._path_copy(self, dst) - @_setdoc(Path) def mkdir(self, mode=None, parents=True, exist_ok=True): if parents and exist_ok: self.remote._path_mkdir(self, mode=mode, minus_p=True) @@ -258,34 +236,28 @@ def mkdir(self, mode=None, parents=True, exist_ok=True): else: raise - @_setdoc(Path) def read(self, encoding=None): data = self.remote._path_read(self) if encoding: data = data.decode(encoding) return data - @_setdoc(Path) def write(self, data, encoding=None): if encoding: data = data.encode(encoding) self.remote._path_write(self, data) - @_setdoc(Path) def touch(self): self.remote._path_touch(str(self)) - @_setdoc(Path) def chown(self, owner=None, group=None, recursive=None): self.remote._path_chown( self, owner, group, self.is_dir() if recursive is None else recursive ) - @_setdoc(Path) def chmod(self, mode): self.remote._path_chmod(mode, self) - @_setdoc(Path) def access(self, mode=0): mode = self._access_mode_to_flags(mode) res = self.remote._path_stat(self) @@ -294,7 +266,6 @@ def access(self, mode=0): mask = res.st_mode & 0x1FF return ((mask >> 6) & mode) or ((mask >> 3) & mode) - @_setdoc(Path) def link(self, dst): if isinstance(dst, RemotePath): if dst.remote is not self.remote: @@ -306,7 +277,6 @@ def link(self, dst): ) self.remote._path_link(self, dst, False) - @_setdoc(Path) def symlink(self, dst): if isinstance(dst, RemotePath): if dst.remote is not self.remote: @@ -332,24 +302,20 @@ def open(self, mode="r", bufsize=-1): "paths for now" ) - @_setdoc(Path) def as_uri(self, scheme="ssh"): return "{}://{}{}".format( scheme, self.remote._fqhost, urllib.pathname2url(str(self)) ) - @property # type: ignore - @_setdoc(Path) + @property def stem(self): return self.name.rsplit(".")[0] - @property # type: ignore - @_setdoc(Path) + @property def root(self): return "/" - @property # type: ignore - @_setdoc(Path) + @property def drive(self): return "" From 72e8c1dbac47f93e714ff36f6b50f16f5e0e8571 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 27 Jan 2022 17:55:53 -0500 Subject: [PATCH 05/37] refactor: use pylint and fix warnings (#578) * refactor: use pylint and fix warnings * fix: some cleanup on top --- .github/workflows/ci.yml | 2 + noxfile.py | 10 +++ plumbum/__init__.py | 14 ++-- plumbum/cli/application.py | 78 +++++++++--------- plumbum/cli/config.py | 7 +- plumbum/cli/i18n.py | 14 ++-- plumbum/cli/image.py | 15 ++-- plumbum/cli/progress.py | 28 +++---- plumbum/cli/switches.py | 69 +++++++--------- plumbum/cli/terminal.py | 35 ++++---- plumbum/colorlib/__init__.py | 5 +- plumbum/colorlib/_ipython_ext.py | 4 +- plumbum/colorlib/factories.py | 20 ++--- plumbum/colorlib/styles.py | 117 ++++++++++++--------------- plumbum/commands/base.py | 15 ++-- plumbum/commands/daemons.py | 19 +++-- plumbum/commands/modifiers.py | 98 +++++++++++----------- plumbum/commands/processes.py | 8 +- plumbum/fs/atomic.py | 11 ++- plumbum/fs/mounts.py | 10 +-- plumbum/lib.py | 16 ++-- plumbum/machines/base.py | 8 +- plumbum/machines/env.py | 8 +- plumbum/machines/local.py | 29 ++++--- plumbum/machines/paramiko_machine.py | 16 ++-- plumbum/machines/remote.py | 74 +++++++++-------- plumbum/machines/session.py | 43 +++++----- plumbum/machines/ssh_machine.py | 10 +-- plumbum/path/base.py | 85 ++++++++++--------- plumbum/path/local.py | 27 ++++--- plumbum/path/remote.py | 49 ++++++----- plumbum/path/utils.py | 39 +++++---- plumbum/typed_env.py | 27 ++++--- pyproject.toml | 41 ++++++++++ 34 files changed, 551 insertions(+), 500 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f5a995dd..a1faba0c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: pre-commit/action@v2.0.3 + - name: pylint + run: pipx run nox -s pylint tests: name: Tests on 🐍 ${{ matrix.python-version }} ${{ matrix.os }} diff --git a/noxfile.py b/noxfile.py index 0ed930890..9b68bfe7a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -16,6 +16,16 @@ def lint(session): session.run("pre-commit", "run", "--all-files", *session.posargs) +@nox.session +def pylint(session): + """ + Run pylint. + """ + + session.install(".", "paramiko", "ipython", "pylint") + session.run("pylint", "plumbum", *session.posargs) + + @nox.session(python=ALL_PYTHONS, reuse_venv=True) def tests(session): """ diff --git a/plumbum/__init__.py b/plumbum/__init__.py index 8e83988e0..a0098a222 100644 --- a/plumbum/__init__.py +++ b/plumbum/__init__.py @@ -35,6 +35,10 @@ See https://plumbum.readthedocs.io for full details """ +import sys +from types import ModuleType +from typing import List + # Avoids a circular import error later import plumbum.path # noqa: F401 from plumbum.commands import ( @@ -83,10 +87,8 @@ # =================================================================================================== # Module hack: ``from plumbum.cmd import ls`` +# Can be replaced by a real module with __getattr__ after Python 3.6 is dropped # =================================================================================================== -import sys -from types import ModuleType -from typing import List class LocalModule(ModuleType): @@ -99,7 +101,7 @@ def __getattr__(self, name): try: return local[name] except CommandNotFound: - raise AttributeError(name) + raise AttributeError(name) from None __path__ = [] # type: List[str] __file__ = __file__ @@ -108,10 +110,6 @@ def __getattr__(self, name): cmd = LocalModule(__name__ + ".cmd", LocalModule.__doc__) sys.modules[cmd.__name__] = cmd -del sys -del ModuleType -del LocalModule - def __dir__(): "Support nice tab completion" diff --git a/plumbum/cli/application.py b/plumbum/cli/application.py index 2b02a2d62..47d6059a4 100644 --- a/plumbum/cli/application.py +++ b/plumbum/cli/application.py @@ -61,7 +61,7 @@ def get(self): try: cls = getattr(mod, clsname) except AttributeError: - raise ImportError(f"cannot import name {clsname}") + raise ImportError(f"cannot import name {clsname}") from None self.subapplication = cls return self.subapplication @@ -173,10 +173,10 @@ def __new__(cls, executable=None): instead of an expression with a dot in it.""" if executable is None: - return cls.run() # This return value was not a class instance, so __init__ is never called - else: - return super().__new__(cls) + return cls.run() + + return super().__new__(cls) def __init__(self, executable): # Filter colors @@ -193,7 +193,7 @@ def __init__(self, executable): # Allow None for the colors self.COLOR_GROUPS = defaultdict( lambda: colors.do_nothing, - dict() if type(self).COLOR_GROUPS is None else type(self).COLOR_GROUPS, + {} if type(self).COLOR_GROUPS is None else type(self).COLOR_GROUPS, ) self.COLOR_GROUP_TITLES = defaultdict( @@ -286,9 +286,8 @@ class FooApp(cli.Application): """ def wrapper(subapp): - attrname = "_subcommand_{}".format( - subapp if isinstance(subapp, str) else subapp.__name__ - ) + subname = subapp if isinstance(subapp, str) else subapp.__name__ + attrname = f"_subcommand_{subname}" setattr(cls, attrname, Subcommand(name, subapp)) return subapp @@ -325,7 +324,7 @@ def _parse_args(self, argv): ) break - elif a.startswith("--") and len(a) >= 3: + if a.startswith("--") and len(a) >= 3: # [--name], [--name=XXX], [--name, XXX], [--name, ==, XXX], # [--name=, XXX], [--name, =XXX] eqsign = a.find("=") @@ -400,12 +399,11 @@ def _parse_args(self, argv): else: if swfuncs[swinfo.func].swname == swname: raise SwitchError(T_("Switch {0} already given").format(swname)) - else: - raise SwitchError( - T_("Switch {0} already given ({1} is equivalent)").format( - swfuncs[swinfo.func].swname, swname - ) + raise SwitchError( + T_("Switch {0} already given ({1} is equivalent)").format( + swfuncs[swinfo.func].swname, swname ) + ) else: if swinfo.list: swfuncs[swinfo.func] = SwitchParseInfo(swname, ([val],), index) @@ -441,7 +439,6 @@ def _parse_args(self, argv): @classmethod def autocomplete(cls, argv): """This is supplied to make subclassing and testing argument completion methods easier""" - pass @staticmethod def _handle_argument(val, argtype, name): @@ -454,7 +451,7 @@ def _handle_argument(val, argtype, name): T_( "Argument of {name} expected to be {argtype}, not {val!r}:\n {ex!r}" ).format(name=name, argtype=argtype, val=val, ex=ex) - ) + ) from None else: return NotImplemented @@ -515,7 +512,7 @@ def _validate_args(self, swfuncs, tailargs): min_args, ).format(min_args, tailargs) ) - elif len(tailargs) > max_args: + if len(tailargs) > max_args: raise PositionalArgumentsError( ngettext( "Expected at most {0} positional argument, got {1}", @@ -579,7 +576,11 @@ def _positional_validate(self, args, validator_list, varargs, argnames, varargna return out_args @classmethod - def run(cls, argv=None, exit=True): # @ReservedAssignment + def run( + cls, + argv=None, + exit=True, # pylint: disable=redefined-builtin + ): """ Runs the application, taking the arguments from ``sys.argv`` by default if nothing is passed. If ``exit`` is @@ -673,9 +674,9 @@ def _parse_kwd_args(self, switches): """Parses keywords (positional arguments), used by invoke.""" swfuncs = {} for index, (swname, val) in enumerate(switches.items(), 1): - switch = getattr(type(self), swname) - swinfo = self._switches_by_func[switch._switch_info.func] - if isinstance(switch, CountOf): + switch_local = getattr(type(self), swname) + swinfo = self._switches_by_func[switch_local._switch_info.func] + if isinstance(switch_local, CountOf): p = (range(val),) elif swinfo.list and not hasattr(val, "__iter__"): raise SwitchError( @@ -704,9 +705,10 @@ def main(self, *args): print(T_("------")) self.help() return 1 - else: - print(T_("main() not implemented")) - return 1 + return 0 + + print(T_("main() not implemented")) + return 1 def cleanup(self, retcode): """Called after ``main()`` and all sub-applications have executed, to perform any necessary cleanup. @@ -857,11 +859,7 @@ def wrapped_paragraphs(text, width): for i, d in enumerate(reversed(m.defaults)): tailargs[-i - 1] = f"[{tailargs[-i - 1]}={d}]" if m.varargs: - tailargs.append( - "{}...".format( - m.varargs, - ) - ) + tailargs.append(f"{m.varargs}...") tailargs = " ".join(tailargs) utc = self.COLOR_USAGE_TITLE if self.COLOR_USAGE_TITLE else self.COLOR_USAGE @@ -922,20 +920,20 @@ def switchs(by_groups, show_groups): indentation = "\n" + " " * (cols - wrapper.width) for switch_info, prefix, color in switchs(by_groups, True): - help = switch_info.help # @ReservedAssignment + help_txt = switch_info.help if switch_info.list: - help += T_("; may be given multiple times") + help_txt += T_("; may be given multiple times") if switch_info.mandatory: - help += T_("; required") + help_txt += T_("; required") if switch_info.requires: - help += T_("; requires {0}").format( + help_txt += T_("; requires {0}").format( ", ".join( (("-" if len(switch) == 1 else "--") + switch) for switch in switch_info.requires ) ) if switch_info.excludes: - help += T_("; excludes {0}").format( + help_txt += T_("; excludes {0}").format( ", ".join( (("-" if len(switch) == 1 else "--") + switch) for switch in switch_info.excludes @@ -943,7 +941,7 @@ def switchs(by_groups, show_groups): ) msg = indentation.join( - wrapper.wrap(" ".join(ln.strip() for ln in help.splitlines())) + wrapper.wrap(" ".join(ln.strip() for ln in help_txt.splitlines())) ) if len(prefix) + wrapper.width >= cols: @@ -960,15 +958,17 @@ def switchs(by_groups, show_groups): subapp = subcls.get() doc = subapp.DESCRIPTION if subapp.DESCRIPTION else getdoc(subapp) if self.SUBCOMMAND_HELPMSG: - help = doc + "; " if doc else "" # @ReservedAssignment - help += self.SUBCOMMAND_HELPMSG.format( + help_str = doc + "; " if doc else "" + help_str += self.SUBCOMMAND_HELPMSG.format( parent=self.PROGNAME, sub=name ) else: - help = doc if doc else "" # @ReservedAssignment + help_str = doc if doc else "" msg = indentation.join( - wrapper.wrap(" ".join(ln.strip() for ln in help.splitlines())) + wrapper.wrap( + " ".join(ln.strip() for ln in help_str.splitlines()) + ) ) if len(name) + wrapper.width >= cols: diff --git a/plumbum/cli/config.py b/plumbum/cli/config.py index c128474ab..75ccf5ff1 100644 --- a/plumbum/cli/config.py +++ b/plumbum/cli/config.py @@ -39,7 +39,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): @abstractmethod def read(self): """Read in the linked file""" - pass @abstractmethod def write(self): @@ -49,12 +48,10 @@ def write(self): @abstractmethod def _get(self, option): """Internal get function for subclasses""" - pass @abstractmethod def _set(self, option, value): """Internal set function for subclasses. Must return the value that was set.""" - pass def get(self, option, default=None): "Get an item from the store, returns default if fails" @@ -89,7 +86,7 @@ def read(self): super().read() def write(self): - with open(self.filename, "w") as f: + with open(self.filename, "w", encoding="utf-8") as f: self.parser.write(f) super().write() @@ -107,7 +104,7 @@ def _get(self, option): try: return self.parser.get(sec, option) except (NoSectionError, NoOptionError): - raise KeyError(f"{sec}:{option}") + raise KeyError(f"{sec}:{option}") from None def _set(self, option, value): sec, option = self._sec_opt(option) diff --git a/plumbum/cli/i18n.py b/plumbum/cli/i18n.py index eec6ed307..4bcd6616b 100644 --- a/plumbum/cli/i18n.py +++ b/plumbum/cli/i18n.py @@ -5,16 +5,18 @@ if loc is None or loc.startswith("en"): class NullTranslation: - def gettext(self, str): - return str + def gettext(self, str1): # pylint: disable=no-self-use + return str1 - def ngettext(self, str1, strN, n): + def ngettext(self, str1, strN, n): # pylint: disable=no-self-use if n == 1: return str1.replace("{0}", str(n)) - else: - return strN.replace("{0}", str(n)) - def get_translation_for(package_name: str) -> NullTranslation: + return strN.replace("{0}", str(n)) + + def get_translation_for( + package_name: str, # pylint: disable=unused-argument + ) -> NullTranslation: return NullTranslation() else: diff --git a/plumbum/cli/image.py b/plumbum/cli/image.py index 246d5724a..ee4ed792c 100644 --- a/plumbum/cli/image.py +++ b/plumbum/cli/image.py @@ -34,18 +34,19 @@ def show(self, filename, double=False): import PIL.Image - if double: - return self.show_pil_double(PIL.Image.open(filename)) - else: - return self.show_pil(PIL.Image.open(filename)) + return ( + self.show_pil_double(PIL.Image.open(filename)) + if double + else self.show_pil(PIL.Image.open(filename)) + ) def _init_size(self, im): """Return the expected image size""" if self.size is None: term_size = get_terminal_size() return self.best_aspect(im.size, term_size) - else: - return self.size + + return self.size def show_pil(self, im): "Standard show routine" @@ -87,7 +88,7 @@ class ShowImageApp(cli.Application): ) @cli.switch(["-c", "--colors"], cli.Range(1, 4), help="Level of color, 1-4") - def colors_set(self, n): + def colors_set(self, n): # pylint: disable=no-self-use colors.use_color = n size = cli.SwitchAttr(["-s", "--size"], help="Size, should be in the form 100x150") diff --git a/plumbum/cli/progress.py b/plumbum/cli/progress.py index 098da97eb..3d21a8a61 100644 --- a/plumbum/cli/progress.py +++ b/plumbum/cli/progress.py @@ -83,7 +83,6 @@ def value(self, val): @abstractmethod def display(self): """Called to update the progress bar""" - pass def increment(self): """Sets next value and displays the bar""" @@ -107,16 +106,15 @@ def str_time_remaining(self): """Returns a string version of time remaining""" if self.value < 1: return "Starting... " - else: - elapsed_time, time_remaining = list(map(str, self.time_remaining())) - return "{} completed, {} remaining".format( - elapsed_time.split(".")[0], time_remaining.split(".")[0] - ) + + elapsed_time, time_remaining = list(map(str, self.time_remaining())) + completed = elapsed_time.split(".")[0] + remaining = time_remaining.split(".")[0] + return f"{completed} completed, {remaining} remaining" @abstractmethod def done(self): """Is called when the iterator is done.""" - pass @classmethod def range(cls, *value, **kargs): @@ -156,15 +154,15 @@ def __str__(self): ) if width - len(ending) < 10 or self.has_output: self.width = 0 + if self.timer: return f"{percent:.0%} complete: {self.str_time_remaining()}" - else: - return f"{percent:.0%} complete" - else: - self.width = width - len(ending) - 2 - 1 - nstars = int(percent * self.width) - pbar = "[" + "*" * nstars + " " * (self.width - nstars) + "]" + ending + return f"{percent:.0%} complete" + + self.width = width - len(ending) - 2 - 1 + nstars = int(percent * self.width) + pbar = "[" + "*" * nstars + " " * (self.width - nstars) + "]" + ending str_percent = f" {percent:.0%} " @@ -243,7 +241,7 @@ class ProgressAuto(ProgressBase): def __new__(cls, *args, **kargs): """Uses the generator trick that if a cls instance is returned, the __init__ method is not called.""" try: # pragma: no cover - __IPYTHON__ + __IPYTHON__ # pylint: disable=pointless-statement try: from traitlets import TraitError except ImportError: # Support for IPython < 4.0 @@ -252,7 +250,7 @@ def __new__(cls, *args, **kargs): try: return ProgressIPy(*args, **kargs) except TraitError: - raise NameError() + raise NameError() from None except (NameError, ImportError): return Progress(*args, **kargs) diff --git a/plumbum/cli/switches.py b/plumbum/cli/switches.py index 68cd9622a..0a9fcfd92 100644 --- a/plumbum/cli/switches.py +++ b/plumbum/cli/switches.py @@ -12,51 +12,35 @@ class SwitchError(Exception): """A general switch related-error (base class of all other switch errors)""" - pass - class PositionalArgumentsError(SwitchError): """Raised when an invalid number of positional arguments has been given""" - pass - class SwitchCombinationError(SwitchError): """Raised when an invalid combination of switches has been given""" - pass - class UnknownSwitch(SwitchError): """Raised when an unrecognized switch has been given""" - pass - class MissingArgument(SwitchError): """Raised when a switch requires an argument, but one was not provided""" - pass - class MissingMandatorySwitch(SwitchError): """Raised when a mandatory switch has not been given""" - pass - class WrongArgumentType(SwitchError): """Raised when a switch expected an argument of some type, but an argument of a wrong type has been given""" - pass - class SubcommandError(SwitchError): """Raised when there's something wrong with sub-commands""" - pass - # =================================================================================================== # The switch decorator @@ -71,11 +55,11 @@ def switch( names, argtype=None, argname=None, - list=False, + list=False, # pylint: disable=redefined-builtin mandatory=False, requires=(), excludes=(), - help=None, + help=None, # pylint: disable=redefined-builtin overridable=False, group="Switches", envname=None, @@ -238,7 +222,13 @@ def main(self): VALUE = _("VALUE") def __init__( - self, names, argtype=str, default=None, list=False, argname=VALUE, **kwargs + self, + names, + argtype=str, + default=None, + list=False, # pylint: disable=redefined-builtin + argname=VALUE, + **kwargs, ): self.__doc__ = "Sets an attribute" # to prevent the help message from showing SwitchAttr's docstring if default and argtype is not None: @@ -266,17 +256,16 @@ def __call__(self, inst, val): def __get__(self, inst, cls): if inst is None: return self - else: - return getattr(inst, self.ATTR_NAME, {}).get(self, self._default_value) + return getattr(inst, self.ATTR_NAME, {}).get(self, self._default_value) def __set__(self, inst, val): if inst is None: raise AttributeError("cannot set an unbound SwitchAttr") + + if not hasattr(inst, self.ATTR_NAME): + setattr(inst, self.ATTR_NAME, {self: val}) else: - if not hasattr(inst, self.ATTR_NAME): - setattr(inst, self.ATTR_NAME, {self: val}) - else: - getattr(inst, self.ATTR_NAME)[self] = val + getattr(inst, self.ATTR_NAME)[self] = val class Flag(SwitchAttr): @@ -369,23 +358,23 @@ def __call__(self, function): m = inspect.getfullargspec(function) args_names = list(m.args[1:]) - positional = [None] * len(args_names) + positional_list = [None] * len(args_names) varargs = None - for i in range(min(len(positional), len(self.args))): - positional[i] = self.args[i] + for i in range(min(len(positional_list), len(self.args))): + positional_list[i] = self.args[i] if len(args_names) + 1 == len(self.args): varargs = self.args[-1] # All args are positional, so convert kargs to positional - for item in self.kargs: + for item, value in self.kargs.items(): if item == m.varargs: - varargs = self.kargs[item] + varargs = value else: - positional[args_names.index(item)] = self.kargs[item] + positional_list[args_names.index(item)] = value - function.positional = positional + function.positional = positional_list function.positional_varargs = varargs return function @@ -397,7 +386,7 @@ class Validator(ABC): def __call__(self, obj): "Must be implemented for a Validator to work" - def choices(self, partial=""): + def choices(self, partial=""): # pylint: disable=no-self-use, unused-argument """Should return set of valid choices, can be given optional partial info""" return set() @@ -409,8 +398,9 @@ def __repr__(self): for prop in getattr(cls, "__slots__", ()): if prop[0] != "_": slots[prop] = getattr(self, prop) - mystrs = (f"{name} = {slots[name]}" for name in slots) - return "{}({})".format(self.__class__.__name__, ", ".join(mystrs)) + mystrs = (f"{name} = {value}" for name, value in slots.items()) + mystrs_str = ", ".join(mystrs) + return f"{self.__class__.__name__}({mystrs_str})" # =================================================================================================== @@ -479,9 +469,8 @@ def __init__(self, *values, **kwargs): self.values = values def __repr__(self): - return "{{{0}}}".format( - ", ".join(v if isinstance(v, str) else v.__name__ for v in self.values) - ) + items = ", ".join(v if isinstance(v, str) else v.__name__ for v in self.values) + return f"{{{items}}}" def __call__(self, value, check_csv=True): if self.csv and check_csv: @@ -523,7 +512,7 @@ def __str__(self): def __call__(self, val): return self.func(val) - def choices(self, partial=""): + def choices(self, partial=""): # pylint: disable=no-self-use, unused-argument return set() @@ -541,7 +530,7 @@ def MakeDirectory(val): p = local.path(val) if p.is_file(): raise ValueError(f"{val} is a file, should be nonexistent, or a directory") - elif not p.exists(): + if not p.exists(): p.mkdir() return p diff --git a/plumbum/cli/terminal.py b/plumbum/cli/terminal.py index fb93f3190..5b8e6d05e 100644 --- a/plumbum/cli/terminal.py +++ b/plumbum/cli/terminal.py @@ -57,14 +57,13 @@ def ask(question, default=None): answer = readline(question).strip().lower() except EOFError: answer = None - if answer in ("y", "yes"): + if answer in {"y", "yes"}: return True - elif answer in ("n", "no"): + if answer in {"n", "no"}: return False - elif not answer and default is not None: + if not answer and default is not None: return default - else: - sys.stdout.write("Invalid response, please try again\n") + sys.stdout.write("Invalid response, please try again\n") def choose(question, options, default=None): @@ -93,7 +92,7 @@ def choose(question, options, default=None): choices = {} defindex = None for i, item in enumerate(options): - i = i + 1 # python2.5 + i += 1 if isinstance(item, (tuple, list)) and len(item) == 2: text = item[0] val = item[1] @@ -103,12 +102,12 @@ def choose(question, options, default=None): choices[i] = val if default is not None and default == val: defindex = i - sys.stdout.write("(%d) %s\n" % (i, text)) + sys.stdout.write(f"({i}) {text}\n") if default is not None: if defindex is None: msg = f"Choice [{default}]: " else: - msg = "Choice [%d]: " % (defindex,) + msg = f"Choice [{defindex}]: " else: msg = "Choice: " while True: @@ -128,7 +127,12 @@ def choose(question, options, default=None): return choices[choice] -def prompt(question, type=str, default=NotImplemented, validator=lambda val: True): +def prompt( + question, + type=str, # pylint: disable=redefined-builtin + default=NotImplemented, + validator=lambda val: True, +): """ Presents the user with a validated question, keeps asking if validation does not pass. @@ -148,22 +152,24 @@ def prompt(question, type=str, default=NotImplemented, validator=lambda val: Tru ans = readline(question).strip() except EOFError: ans = "" + if not ans: if default is not NotImplemented: # sys.stdout.write("\b%s\n" % (default,)) return default - else: - continue + continue try: ans = type(ans) except (TypeError, ValueError) as ex: sys.stdout.write(f"Invalid value ({ex}), please try again\n") continue + try: valid = validator(ans) except ValueError as ex: sys.stdout.write(f"{ex}, please try again\n") continue + if not valid: sys.stdout.write("Value not in specified range, please try again\n") continue @@ -200,11 +206,8 @@ def read_chunk(): prev = chunk if skipped: yield "*" - yield "{:06x} | {}| {}".format( - i * bytes_per_line, - hexd.ljust(bytes_per_line * 3, " "), - text, - ) + hexd_ljust = hexd.ljust(bytes_per_line * 3, " ") + yield f"{i*bytes_per_line:06x} | {hexd_ljust}| {text}" skipped = False diff --git a/plumbum/colorlib/__init__.py b/plumbum/colorlib/__init__.py index b35ee93ce..846e5e0b2 100644 --- a/plumbum/colorlib/__init__.py +++ b/plumbum/colorlib/__init__.py @@ -4,6 +4,7 @@ underlined text. It also provides ``reset`` to recover the normal font. """ +import sys from .factories import StyleFactory from .styles import ANSIStyle, ColorNotFound, HTMLStyle, Style @@ -26,7 +27,7 @@ def load_ipython_extension(ipython): # pragma: no cover try: - from ._ipython_ext import OutputMagics + from ._ipython_ext import OutputMagics # pylint:disable=import-outside-toplevel except ImportError: print("IPython required for the IPython extension to be loaded.") raise @@ -38,8 +39,6 @@ def load_ipython_extension(ipython): # pragma: no cover def main(): # pragma: no cover """Color changing script entry. Call using python -m plumbum.colors, will reset if no arguments given.""" - import sys - color = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else "" ansicolors.use_color = True ansicolors.get_colors_from_string(color).now() diff --git a/plumbum/colorlib/_ipython_ext.py b/plumbum/colorlib/_ipython_ext.py index 707be59e8..f8ff2c306 100644 --- a/plumbum/colorlib/_ipython_ext.py +++ b/plumbum/colorlib/_ipython_ext.py @@ -18,12 +18,12 @@ def to(self, line, cell, local_ns=None): ) display_fn = getattr(IPython.display, "display_" + choice) - "Captures stdout and renders it in the notebook with some ." + # Captures stdout and renders it in the notebook with StringIO() as out: old_out = sys.stdout try: sys.stdout = out - exec(cell, self.shell.user_ns, local_ns) + exec(cell, self.shell.user_ns, local_ns) # pylint: disable=exec-used out.seek(0) display_fn(out.getvalue(), raw=True) finally: diff --git a/plumbum/colorlib/factories.py b/plumbum/colorlib/factories.py index 21e3bc52b..34cdd4e9c 100644 --- a/plumbum/colorlib/factories.py +++ b/plumbum/colorlib/factories.py @@ -33,7 +33,7 @@ def __getattr__(self, item): try: return self._style.from_color(self._style.color_class(item, fg=self._fg)) except ColorNotFound: - raise AttributeError(item) + raise AttributeError(item) from None def full(self, name): """Gets the style for a color, using standard name procedure: either full @@ -52,8 +52,8 @@ def rgb(self, r, g=None, b=None): """Return the extended color scheme color for a value.""" if g is None and b is None: return self.hex(r) - else: - return self._style.from_color(self._style.color_class(r, g, b, fg=self._fg)) + + return self._style.from_color(self._style.color_class(r, g, b, fg=self._fg)) def hex(self, hexcode): """Return the extended color scheme color for a value.""" @@ -73,9 +73,10 @@ def __getitem__(self, val): (start, stop, stride) = val.indices(256) if stop <= 16: return [self.simple(v) for v in range(start, stop, stride)] - else: - return [self.full(v) for v in range(start, stop, stride)] - elif isinstance(val, tuple): + + return [self.full(v) for v in range(start, stop, stride)] + + if isinstance(val, tuple): return self.rgb(*val) try: @@ -107,13 +108,12 @@ def __enter__(self): """This will reset the color on leaving the with statement.""" return self - def __exit__(self, type, value, traceback): + def __exit__(self, _type, _value, _traceback) -> None: """This resets a FG/BG color or all styles, due to different definition of RESET for the factories.""" self.reset.now() - return False def __repr__(self): """Simple representation of the class by name.""" @@ -200,6 +200,8 @@ def extract(self, colored_string): """Gets colors from an ansi string, returns those colors""" return self._style.from_ansi(colored_string, True) - def load_stylesheet(self, stylesheet=default_styles): + def load_stylesheet(self, stylesheet=None): + if stylesheet is None: + stylesheet = default_styles for item in stylesheet: setattr(self, item, self.get_colors_from_string(stylesheet[item])) diff --git a/plumbum/colorlib/styles.py b/plumbum/colorlib/styles.py index e007944af..38db87313 100644 --- a/plumbum/colorlib/styles.py +++ b/plumbum/colorlib/styles.py @@ -41,7 +41,7 @@ def get_color_repr(): """Gets best colors for current system.""" if "NO_COLOR" in os.environ: return 0 - if os.environ.get("FORCE_COLOR", "0") in {"0", "1", "2", "3", "4"}: + if os.environ.get("FORCE_COLOR", "") in {"0", "1", "2", "3", "4"}: return int(os.environ["FORCE_COLOR"]) if not sys.stdout.isatty(): return 0 @@ -64,21 +64,15 @@ def get_color_repr(): class ColorNotFound(Exception): """Thrown when a color is not valid for a particular method.""" - pass - class AttributeNotFound(Exception): """Similar to color not found, only for attributes.""" - pass - class ResetNotSupported(Exception): """An exception indicating that Reset is not available for this Style.""" - pass - class Color(ABC): """\ @@ -203,7 +197,7 @@ def _from_simple(self, color): if color == "reset": return - elif color in _lower_camel_names[:16]: + if color in _lower_camel_names[:16]: self.number = _lower_camel_names.index(color) self.rgb = from_html(color_html[self.number]) @@ -235,7 +229,7 @@ def _from_full(self, color): if color == "reset": return - elif color in _lower_camel_names: + if color in _lower_camel_names: self.number = _lower_camel_names.index(color) self.rgb = from_html(color_html[self.number]) @@ -261,7 +255,7 @@ def _from_hex(self, color): try: self.rgb = from_html(color) except (TypeError, ValueError): - raise ColorNotFound("Did not find htmlcode: " + repr(color)) + raise ColorNotFound("Did not find htmlcode: " + repr(color)) from None self.representation = 4 self._init_number() @@ -269,10 +263,7 @@ def _from_hex(self, color): @property def name(self): """The (closest) name of the current color""" - if self.isreset: - return "reset" - else: - return color_names[self.number] + return "reset" if self.isreset else color_names[self.number] @property def name_camelcase(self): @@ -289,10 +280,7 @@ def __repr__(self): def __eq__(self, other): """Reset colors are equal, otherwise rgb have to match.""" - if self.isreset: - return other.isreset - else: - return self.rgb == other.rgb + return other.isreset if self.isreset else self.rgb == other.rgb @property def ansi_sequence(self): @@ -306,20 +294,22 @@ def ansi_codes(self): if self.isreset: return (ansi_addition + 9,) - elif self.representation < 3: + if self.representation < 3: return (color_codes_simple[self.number] + ansi_addition,) - elif self.representation == 3: + if self.representation == 3: return (ansi_addition + 8, 5, self.number) - else: - return (ansi_addition + 8, 2, self.rgb[0], self.rgb[1], self.rgb[2]) + + return (ansi_addition + 8, 2, self.rgb[0], self.rgb[1], self.rgb[2]) @property def hex_code(self): """This is the hex code of the current color, html style notation.""" - if self.isreset: - return "#000000" - else: - return "#" + "{0[0]:02X}{0[1]:02X}{0[2]:02X}".format(self.rgb) + + return ( + "#000000" + if self.isreset + else f"#{self.rgb[0]:02X}{self.rgb[1]:02X}{self.rgb[2]:02X}" + ) def __str__(self): """This just prints it's simple name""" @@ -338,10 +328,7 @@ def to_representation(self, val): def limit_representation(self, val): """Only converts if val is lower than representation""" - if self.representation <= val: - return self - else: - return self.to_representation(val) + return self if self.representation <= val else self.to_representation(val) class Style: @@ -372,7 +359,7 @@ def stdout(self): Unfortunately, it only works on an instance.. """ # Import sys repeated here to make calling this stable in atexit function - import sys + import sys # pylint: disable=reimported, redefined-outer-name, import-outside-toplevel return ( self.__class__._stdout if self.__class__._stdout is not None else sys.stdout @@ -388,7 +375,7 @@ def __init__(self, attributes=None, fgcolor=None, bgcolor=None, reset=False): for item in ("attributes", "fg", "bg", "isreset"): setattr(self, item, copy(getattr(attributes, item))) return - self.attributes = attributes if attributes is not None else dict() + self.attributes = attributes if attributes is not None else {} self.fg = fgcolor self.bg = bgcolor self.isreset = reset @@ -400,10 +387,7 @@ def __init__(self, attributes=None, fgcolor=None, bgcolor=None, reset=False): @classmethod def from_color(cls, color): - if color.fg: - self = cls(fgcolor=color) - else: - self = cls(bgcolor=color) + self = cls(fgcolor=color) if color.fg else cls(bgcolor=color) return self def invert(self): @@ -468,8 +452,8 @@ def __add__(self, other): if not result.bg: result.bg = self.bg return result - else: - return other.__class__(self) + other + + return other.__class__(self) + other def __radd__(self, other): """This only gets called if the string is on the left side. (Not safe)""" @@ -484,8 +468,8 @@ def __and__(self, other): and ``color & "String" syntax too.``""" if type(self) == type(other): return self + other - else: - return self.wrap(other) + + return self.wrap(other) def __rand__(self, other): """This class supports ``"String:" & color`` syntax.""" @@ -536,7 +520,7 @@ def __enter__(self): self.stdout.write(str(self)) self.stdout.flush() - def __exit__(self, type, value, traceback): + def __exit__(self, _type, _value, _traceback): """Runs even if exception occurred, does not catch it.""" self.stdout.write(str(~self)) self.stdout.flush() @@ -573,37 +557,37 @@ def ansi_codes(self): @property def ansi_sequence(self): """This is the string ANSI sequence.""" - codes = self.ansi_codes - if codes: - return "\033[" + ";".join(map(str, self.ansi_codes)) + "m" - else: - return "" + codes = ";".join(str(c) for c in self.ansi_codes) + return f"\033[{codes}m" if codes else "" def __repr__(self): name = self.__class__.__name__ attributes = ", ".join(a for a in self.attributes if self.attributes[a]) neg_attributes = ", ".join( - "-" + a for a in self.attributes if not self.attributes[a] + f"-{a}" for a in self.attributes if not self.attributes[a] ) colors = ", ".join(repr(c) for c in [self.fg, self.bg] if c) - string = "; ".join(s for s in [attributes, neg_attributes, colors] if s) + string = ( + "; ".join(s for s in [attributes, neg_attributes, colors] if s) or "empty" + ) if self.isreset: string = "reset" - return "<{}: {}>".format(name, string if string else "empty") + return f"<{name}: {string}>" def __eq__(self, other): """Equality is true only if reset, or if attributes, fg, and bg match.""" if type(self) == type(other): + if self.isreset: return other.isreset - else: - return ( - self.attributes == other.attributes - and self.fg == other.fg - and self.bg == other.bg - ) - else: - return str(self) == other + + return ( + self.attributes == other.attributes + and self.fg == other.fg + and self.bg == other.bg + ) + + return str(self) == other @abstractmethod def __str__(self): @@ -627,7 +611,7 @@ def add_ansi(self, sequence, filter_resets=False): try: while True: value = next(values) - if value == 38 or value == 48: + if value in {38, 48}: fg = value == 38 value = next(values) if value == 5: @@ -650,13 +634,13 @@ def add_ansi(self, sequence, filter_resets=False): if filter_resets is False: self.isreset = True elif value in attributes_ansi.values(): - for name in attributes_ansi: - if value == attributes_ansi[name]: + for name, att_value in attributes_ansi.items(): + if value == att_value: self.attributes[name] = True elif value in (20 + n for n in attributes_ansi.values()): if filter_resets is False: - for name in attributes_ansi: - if value == attributes_ansi[name] + 20: + for name, att_value in attributes_ansi.items(): + if value == att_value + 20: self.attributes[name] = False elif 30 <= value <= 37: self.fg = self.color_class.from_simple(value - 30) @@ -743,10 +727,11 @@ class ANSIStyle(Style): attribute_names = attributes_ansi def __str__(self): - if not self.use_color: - return "" - else: - return self.limit_representation(self.use_color).ansi_sequence + return ( + self.limit_representation(self.use_color).ansi_sequence + if self.use_color + else "" + ) class HTMLStyle(Style): diff --git a/plumbum/commands/base.py b/plumbum/commands/base.py index 09edacba7..ec199c573 100644 --- a/plumbum/commands/base.py +++ b/plumbum/commands/base.py @@ -101,10 +101,11 @@ def bound_command(self, *args): """Creates a bound-command with the given arguments""" if not args: return self + if isinstance(self, BoundCommand): return BoundCommand(self.cmd, self.args + list(args)) - else: - return BoundCommand(self, args) + + return BoundCommand(self, args) def __call__(self, *args, **kwargs): """A shortcut for `run(args)`, returning only the process' stdout""" @@ -207,7 +208,7 @@ def bgrun(self, args=(), **kwargs): def runner(): if was_run[0]: - return # already done + return None # already done was_run[0] = True try: return run_proc(p, retcode, timeout) @@ -333,7 +334,7 @@ def popen(self, args=(), **kwargs): class BoundEnvCommand(BaseCommand): - __slots__ = ("cmd", "env", "cwd") + __slots__ = ("cmd",) def __init__(self, cmd, env=None, cwd=None): self.cmd = cmd @@ -441,7 +442,7 @@ class BaseRedirection(BaseCommand): __slots__ = ("cmd", "file") SYM = None # type: str KWARG = None # type: str - MODE = None # type: str + MODE = "r" # type: str def __init__(self, cmd, file): self.cmd = cmd @@ -471,8 +472,8 @@ def popen(self, args=(), **kwargs): raise RedirectionError(f"{self.KWARG} is already redirected") if isinstance(self.file, RemotePath): raise TypeError("Cannot redirect to/from remote paths") - if isinstance(self.file, (str,) + (LocalPath,)): - f = kwargs[self.KWARG] = open(str(self.file), self.MODE) + if isinstance(self.file, (str, LocalPath)): + f = kwargs[self.KWARG] = open(str(self.file), self.MODE, encoding="utf-8") else: kwargs[self.KWARG] = self.file f = None diff --git a/plumbum/commands/daemons.py b/plumbum/commands/daemons.py index 1fcf259d3..4c1de6bb6 100644 --- a/plumbum/commands/daemons.py +++ b/plumbum/commands/daemons.py @@ -12,13 +12,16 @@ class _fake_lock: """Needed to allow normal os.exit() to work without error""" - def acquire(self, val): + @staticmethod + def acquire(_): return True - def release(self): + @staticmethod + def release(): pass +# pylint: disable-next: inconsistent-return-statements def posix_daemonize(command, cwd, stdout=None, stderr=None, append=True): if stdout is None: stdout = os.devnull @@ -36,9 +39,9 @@ def posix_daemonize(command, cwd, stdout=None, stderr=None, append=True): try: os.setsid() os.umask(0) - stdin = open(os.devnull) - stdout = open(stdout, "a" if append else "w") - stderr = open(stderr, "a" if append else "w") + stdin = open(os.devnull, encoding="utf-8") + stdout = open(stdout, "a" if append else "w", encoding="utf-8") + stderr = open(stderr, "a" if append else "w", encoding="utf-8") signal.signal(signal.SIGHUP, signal.SIG_IGN) proc = command.popen( cwd=cwd, @@ -113,9 +116,9 @@ def win32_daemonize(command, cwd, stdout=None, stderr=None, append=True): if stderr is None: stderr = stdout DETACHED_PROCESS = 0x00000008 - stdin = open(os.devnull) - stdout = open(stdout, "a" if append else "w") - stderr = open(stderr, "a" if append else "w") + stdin = open(os.devnull, encoding="utf-8") + stdout = open(stdout, "a" if append else "w", encoding="utf-8") + stderr = open(stderr, "a" if append else "w", encoding="utf-8") return command.popen( cwd=cwd, stdin=stdin.fileno(), diff --git a/plumbum/commands/modifiers.py b/plumbum/commands/modifiers.py index 6287af201..208229116 100644 --- a/plumbum/commands/modifiers.py +++ b/plumbum/commands/modifiers.py @@ -1,4 +1,5 @@ import sys +from logging import DEBUG, INFO from select import select from subprocess import PIPE @@ -22,10 +23,8 @@ def __init__(self, proc, expected_retcode, timeout=None): self._stderr = None def __repr__(self): - return "".format( - self.proc.argv, - self._returncode if self.ready() else "running", - ) + running = self._returncode if self.ready() else "running" + return f"" def poll(self): """Polls the underlying process for termination; returns ``False`` if still running, @@ -83,8 +82,9 @@ def __repr__(self): for prop in slots_list: if prop[0] != "_": slots[prop] = getattr(self, prop) - mystrs = (f"{name} = {slots[name]}" for name in slots) - return "{}({})".format(self.__class__.__name__, ", ".join(mystrs)) + mystrs = (f"{name} = {value}" for name, value in slots.items()) + mystrs_str = ", ".join(mystrs) + return f"{self.__class__.__name__}({mystrs_str})" @classmethod def __call__(cls, *args, **kwargs): @@ -121,9 +121,6 @@ def __rand__(self, cmd): return Future(cmd.popen(**self.kargs), self.retcode, timeout=self.timeout) -BG = _BG() - - class _FG(ExecutionModifier): """ An execution modifier that runs the given command in the foreground, passing it the @@ -154,9 +151,6 @@ def __rand__(self, cmd): ) -FG = _FG() - - class _TEE(ExecutionModifier): """Run a command, dumping its stdout/stderr to the current process's stdout and stderr, but ALSO return them. Useful for interactive programs that @@ -237,9 +231,6 @@ def __rand__(self, cmd): return p.returncode, stdout, stderr -TEE = _TEE() - - class _TF(ExecutionModifier): """ An execution modifier that runs the given command, but returns True/False depending on the retcode. @@ -259,7 +250,12 @@ class _TF(ExecutionModifier): __slots__ = ("retcode", "FG", "timeout") - def __init__(self, retcode=0, FG=False, timeout=None): + def __init__( + self, + retcode=0, + FG=False, # pylint: disable=redefined-outer-name + timeout=None, + ): """`retcode` is the return code to expect to mean "success". Set `FG` to True to run in the foreground. """ @@ -288,9 +284,6 @@ def __rand__(self, cmd): return False -TF = _TF() - - class _RETCODE(ExecutionModifier): """ An execution modifier that runs the given command, causing it to run and return the retcode. @@ -307,7 +300,11 @@ class _RETCODE(ExecutionModifier): __slots__ = ("foreground", "timeout") - def __init__(self, FG=False, timeout=None): + def __init__( + self, + FG=False, # pylint: disable=redefined-outer-name + timeout=None, + ): """`FG` to True to run in the foreground.""" self.foreground = FG self.timeout = timeout @@ -318,14 +315,12 @@ def __call__(cls, *args, **kwargs): def __rand__(self, cmd): if self.foreground: - return cmd.run( + result = cmd.run( retcode=None, stdin=None, stdout=None, stderr=None, timeout=self.timeout - )[0] - else: - return cmd.run(retcode=None, timeout=self.timeout)[0] + ) + return result[0] - -RETCODE = _RETCODE() + return cmd.run(retcode=None, timeout=self.timeout)[0] class _NOHUP(ExecutionModifier): @@ -377,7 +372,27 @@ def __rand__(self, cmd): return cmd.nohup(self.cwd, stdout, self.stderr, append) -NOHUP = _NOHUP() +class LogPipe: + def __init__(self, line_timeout, kw, levels, prefix, log): + self.line_timeout = line_timeout + self.kw = kw + self.levels = levels + self.prefix = prefix + self.log = log + + def __rand__(self, cmd): + popen = cmd if hasattr(cmd, "iter_lines") else cmd.popen() + for typ, lines in popen.iter_lines( + line_timeout=self.line_timeout, mode=BY_TYPE, **self.kw + ): + if not lines: + continue + level = self.levels[typ] + for line in lines.splitlines(): + if self.prefix: + line = f"{self.prefix}: {line}" + self.log(level, line) + return popen.returncode class PipeToLoggerMixin: @@ -424,11 +439,11 @@ class MyLogger(logbook.Logger, PipeToLoggerMixin): """ - from logging import DEBUG, INFO - DEFAULT_LINE_TIMEOUT = 10 * 60 DEFAULT_STDOUT = "INFO" DEFAULT_STDERR = "DEBUG" + INFO = INFO + DEBUG = DEBUG def pipe( self, out_level=None, err_level=None, prefix=None, line_timeout=None, **kw @@ -442,21 +457,6 @@ def pipe( Optionally use `prefix` for each line. """ - class LogPipe: - def __rand__(_, cmd): - popen = cmd if hasattr(cmd, "iter_lines") else cmd.popen() - for typ, lines in popen.iter_lines( - line_timeout=line_timeout, mode=BY_TYPE, **kw - ): - if not lines: - continue - level = levels[typ] - for line in lines.splitlines(): - if prefix: - line = f"{prefix}: {line}" - self.log(level, line) - return popen.returncode - levels = { 1: getattr(self, self.DEFAULT_STDOUT), 2: getattr(self, self.DEFAULT_STDERR), @@ -471,7 +471,7 @@ def __rand__(_, cmd): if err_level is not None: levels[2] = err_level - return LogPipe() + return LogPipe(line_timeout, kw, levels, prefix, self.log) def pipe_info(self, prefix=None, **kw): """ @@ -493,3 +493,11 @@ def __rand__(self, cmd): return cmd & self.pipe( getattr(self, self.DEFAULT_STDOUT), getattr(self, self.DEFAULT_STDERR) ) + + +BG = _BG() +FG = _FG() +NOHUP = _NOHUP() +RETCODE = _RETCODE() +TEE = _TEE() +TF = _TF() diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 58b993891..068b7a7d2 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -49,7 +49,7 @@ def selector(): def _iter_lines_win32(proc, decode, linesize, line_timeout=None): class Piper(Thread): def __init__(self, fd, pipe): - super().__init__(name="PlumbumPiper%sThread" % fd) + super().__init__(name=f"PlumbumPiper{fd}Thread") self.pipe = pipe self.fd = fd self.empty = False @@ -115,7 +115,7 @@ class ProcessExecutionError(EnvironmentError): """ def __init__(self, argv, retcode, stdout, stderr, message=None): - Exception.__init__(self, argv, retcode, stdout, stderr) + super().__init__(self, argv, retcode, stdout, stderr) self.message = message self.argv = argv self.retcode = retcode @@ -172,7 +172,7 @@ class CommandNotFound(AttributeError): command was not found in the system's ``PATH``""" def __init__(self, program, path): - Exception.__init__(self, program, path) + super().__init__(self, program, path) self.program = program self.path = path @@ -250,7 +250,7 @@ def _register_proc_timeout(proc, timeout): def _shutdown_bg_threads(): - global _shutting_down + global _shutting_down # pylint: disable=global-statement _shutting_down = True # Make sure this still exists (don't throw error in atexit!) if _timeout_queue: diff --git a/plumbum/fs/atomic.py b/plumbum/fs/atomic.py index d8baa392e..16954c716 100644 --- a/plumbum/fs/atomic.py +++ b/plumbum/fs/atomic.py @@ -20,9 +20,8 @@ from win32con import LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY from win32file import OVERLAPPED, LockFileEx, UnlockFile except ImportError: - raise ImportError( - "On Windows, we require Python for Windows Extensions (pywin32)" - ) + print("On Windows, Plumbum requires Python for Windows Extensions (pywin32)") + raise @contextmanager def locked_file(fileno, blocking=True): @@ -38,7 +37,7 @@ def locked_file(fileno, blocking=True): ) except WinError: _, ex, _ = sys.exc_info() - raise OSError(*ex.args) + raise OSError(*ex.args) from None try: yield finally: @@ -277,7 +276,7 @@ def __exit__(self, t, v, tb): def __del__(self): try: self.release() - except Exception: + except Exception: # pylint:disable=broad-except pass def close(self): @@ -303,7 +302,7 @@ def acquire(self): raise PidFileTaken( f"PID file {self.atomicfile.path!r} taken by process {pid}", pid, - ) + ) from None else: self.atomicfile.write_atomic(str(os.getpid()).encode("utf8")) atexit.register(self.release) diff --git a/plumbum/fs/mounts.py b/plumbum/fs/mounts.py index 359919a01..c8c761324 100644 --- a/plumbum/fs/mounts.py +++ b/plumbum/fs/mounts.py @@ -13,12 +13,8 @@ def __init__(self, dev, point, fstype, options): self.options = options.split(",") def __str__(self): - return "{} on {} type {} ({})".format( - self.dev, - self.point, - self.fstype, - ",".join(self.options), - ) + options = ",".join(self.options) + return f"{self.dev} on {self.point} type {self.fstype} ({options})" MOUNT_PATTERN = re.compile(r"(.+?)\s+on\s+(.+?)\s+type\s+(\S+)(?:\s+\((.+?)\))?") @@ -42,4 +38,4 @@ def mounted(fs): """ Indicates if a the given filesystem (device file or mount point) is currently mounted """ - return any(fs == entry.dev or fs == entry.point for entry in mount_table()) + return any(fs in {entry.dev, entry.point} for entry in mount_table()) diff --git a/plumbum/lib.py b/plumbum/lib.py index 0a2f7b174..9a489de57 100644 --- a/plumbum/lib.py +++ b/plumbum/lib.py @@ -15,9 +15,7 @@ def __init__(self, pid, uid, stat, args): self.args = args def __repr__(self): - return "ProcInfo({!r}, {!r}, {!r}, {!r})".format( - self.pid, self.uid, self.stat, self.args - ) + return f"ProcInfo({self.pid!r}, {self.uid!r}, {self.stat!r}, {self.args!r})" @contextmanager @@ -48,13 +46,13 @@ def __get__(self, obj, klass=None): return self._function() -def getdoc(object): +def getdoc(obj): """ This gets a docstring if available, and cleans it, but does not look up docs in - inheritance tree (Pre 3.5 behavior of ``inspect.getdoc``). + inheritance tree (Pre Python 3.5 behavior of ``inspect.getdoc``). """ try: - doc = object.__doc__ + doc = obj.__doc__ except AttributeError: return None if not isinstance(doc, str): @@ -70,12 +68,12 @@ def read_fd_decode_safely(fd, size=4096): Returns the data and the decoded text. """ data = os.read(fd.fileno(), size) - for i in range(4): + for _ in range(3): try: return data, data.decode("utf-8") except UnicodeDecodeError as e: if e.reason != "unexpected end of data": raise - if i == 3: - raise data += os.read(fd.fileno(), 1) + + return data, data.decode("utf-8") diff --git a/plumbum/machines/base.py b/plumbum/machines/base.py index e9d9b62b8..a498c7c2f 100644 --- a/plumbum/machines/base.py +++ b/plumbum/machines/base.py @@ -49,13 +49,11 @@ def get(self, cmd, *othercommands): command = self[cmd] if not command.executable.exists(): raise CommandNotFound(cmd, command.executable) - else: - return command + return command except CommandNotFound: if othercommands: return self.get(othercommands[0], *othercommands[1:]) - else: - raise + raise def __contains__(self, cmd): """Tests for the existence of the command, e.g., ``"ls" in plumbum.local``. @@ -88,7 +86,7 @@ def __getattr__(self, name): try: return self._machine[name] except CommandNotFound: - raise AttributeError(name) + raise AttributeError(name) from None @property def cmd(self): diff --git a/plumbum/machines/env.py b/plumbum/machines/env.py index 2af836bd5..a430632e0 100644 --- a/plumbum/machines/env.py +++ b/plumbum/machines/env.py @@ -6,6 +6,7 @@ class EnvPathList(list): __slots__ = ["_path_factory", "_pathsep", "__weakref__"] def __init__(self, path_factory, pathsep): + super().__init__() self._path_factory = path_factory self._pathsep = pathsep @@ -40,7 +41,8 @@ class BaseEnv: __slots__ = ["_curr", "_path", "_path_factory", "__weakref__"] CASE_SENSITIVE = True - def __init__(self, path_factory, pathsep): + def __init__(self, path_factory, pathsep, *, _curr): + self._curr = _curr self._path_factory = path_factory self._path = EnvPathList(path_factory, pathsep) self._update_path() @@ -150,9 +152,9 @@ def path(self): def _get_home(self): if "HOME" in self: return self._path_factory(self["HOME"]) - elif "USERPROFILE" in self: # pragma: no cover + if "USERPROFILE" in self: # pragma: no cover return self._path_factory(self["USERPROFILE"]) - elif "HOMEPATH" in self: # pragma: no cover + if "HOMEPATH" in self: # pragma: no cover return self._path_factory(self.get("HOMEDRIVE", ""), self["HOMEPATH"]) return None diff --git a/plumbum/machines/local.py b/plumbum/machines/local.py index 80db95701..723c72c53 100644 --- a/plumbum/machines/local.py +++ b/plumbum/machines/local.py @@ -24,7 +24,7 @@ class PlumbumLocalPopen(PopenAddons): iter_lines = iter_lines def __init__(self, *args, **kwargs): - self._proc = Popen(*args, **kwargs) + self._proc = Popen(*args, **kwargs) # pylint: disable=consider-using-with def __iter__(self): return self.iter_lines() @@ -56,8 +56,7 @@ class LocalEnv(BaseEnv): def __init__(self): # os.environ already takes care of upper'ing on windows - self._curr = os.environ.copy() - BaseEnv.__init__(self, LocalPath, os.path.pathsep) + super().__init__(LocalPath, os.path.pathsep, _curr=os.environ.copy()) if IS_WIN32 and "HOME" not in self and self.home is not None: self["HOME"] = self.home @@ -223,15 +222,15 @@ def __getitem__(self, cmd): if isinstance(cmd, LocalPath): return LocalCommand(cmd) - elif not isinstance(cmd, RemotePath): + + if not isinstance(cmd, RemotePath): if "/" in cmd or "\\" in cmd: # assume path return LocalCommand(local.path(cmd)) - else: - # search for command - return LocalCommand(self.which(cmd)) - else: - raise TypeError(f"cmd must not be a RemotePath: {cmd!r}") + # search for command + return LocalCommand(self.which(cmd)) + + raise TypeError(f"cmd must not be a RemotePath: {cmd!r}") def _popen( self, @@ -317,12 +316,12 @@ def daemonic_popen(self, command, cwd="/", stdout=None, stderr=None, append=True """ if IS_WIN32: return win32_daemonize(command, cwd, stdout, stderr, append) - else: - return posix_daemonize(command, cwd, stdout, stderr, append) + + return posix_daemonize(command, cwd, stdout, stderr, append) if IS_WIN32: - def list_processes(self): + def list_processes(self): # pylint: disable=no-self-use """ Returns information about all running processes (on Windows: using ``tasklist``) @@ -379,11 +378,11 @@ def session(self, new_session=False): def tempdir(self): """A context manager that creates a temporary directory, which is removed when the context exits""" - dir = self.path(mkdtemp()) # @ReservedAssignment + new_dir = self.path(mkdtemp()) try: - yield dir + yield new_dir finally: - dir.delete() + new_dir.delete() @contextmanager def as_user(self, username=None): diff --git a/plumbum/machines/paramiko_machine.py b/plumbum/machines/paramiko_machine.py index 9cd7c6a2b..3a5858e00 100644 --- a/plumbum/machines/paramiko_machine.py +++ b/plumbum/machines/paramiko_machine.py @@ -70,7 +70,8 @@ def close(self): self.channel.shutdown_write() self.channel.close() - def kill(self): + @staticmethod + def kill(): # possible way to obtain pid: # "(cmd ; echo $?) & echo ?!" # and then client.exec_command("kill -9 %s" % (pid,)) @@ -106,7 +107,7 @@ def communicate(self): self.stdin.flush() i = (i + 1) % len(sources) - name, coll, pipe, outfile = sources[i] + _name, coll, pipe, outfile = sources[i] line = pipe.readline() # logger.debug("%s> %r", name, line) if not line: @@ -255,7 +256,7 @@ def __init__( kwargs["gss_host"] = gss_host if load_system_ssh_config: ssh_config = paramiko.SSHConfig() - with open(os.path.expanduser("~/.ssh/config")) as f: + with open(os.path.expanduser("~/.ssh/config"), encoding="utf-8") as f: ssh_config.parse(f) try: hostConfig = ssh_config.lookup(host) @@ -308,7 +309,7 @@ def popen( stdin=None, stdout=None, stderr=None, - new_session=False, + new_session=False, # pylint: disable=unused-argument env=None, cwd=None, ): @@ -474,7 +475,12 @@ def recv(self, count): ################################################################################################### # Custom iter_lines for paramiko.Channel ################################################################################################### -def _iter_lines(proc, decode, linesize, line_timeout=None): +def _iter_lines( + proc, + decode, # pylint: disable=unused-argument + linesize, + line_timeout=None, +): from selectors import EVENT_READ, DefaultSelector diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index d69a9c753..d379765ac 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -16,28 +16,29 @@ class RemoteEnv(BaseEnv): __slots__ = ["_orig", "remote"] def __init__(self, remote): - self.remote = remote session = remote._session # GNU env has a -0 argument; use it if present. Otherwise, # fall back to calling printenv on each (possible) variable # from plain env. env0 = session.run("env -0; echo") if env0[0] == 0 and not env0[2].rstrip(): - self._curr = dict( + _curr = dict( line.split("=", 1) for line in env0[1].split("\x00") if "=" in line ) else: lines = session.run("env; echo")[1].splitlines() split = (line.split("=", 1) for line in lines) keys = (line[0] for line in split if len(line) > 1) - runs = ((key, session.run('printenv "%s"; echo' % key)) for key in keys) - self._curr = { + runs = ((key, session.run(f'printenv "{key}"; echo')) for key in keys) + _curr = { key: run[1].rstrip("\n") for (key, run) in runs if run[0] == 0 and run[1].rstrip("\n") and not run[2] } + + super().__init__(remote.path, ":", _curr=_curr) + self.remote = remote self._orig = self._curr.copy() - BaseEnv.__init__(self, self.remote.path, ":") def __delitem__(self, name): BaseEnv.__delitem__(self, name) @@ -177,15 +178,15 @@ def _get_uname(self): rc, out, _ = self._session.run("uname", retcode=None) if rc == 0: return out.strip() - else: - rc, out, _ = self._session.run( - "python -c 'import platform;print(platform.uname()[0])'", retcode=None - ) - if rc == 0: - return out.strip() - else: - # all POSIX systems should have uname. make an educated guess it's Windows - return "Windows" + + rc, out, _ = self._session.run( + "python -c 'import platform;print(platform.uname()[0])'", retcode=None + ) + if rc == 0: + return out.strip() + + # all POSIX systems should have uname. make an educated guess it's Windows + return "Windows" def __repr__(self): return f"<{self.__class__.__name__} {self}>" @@ -247,19 +248,17 @@ def __getitem__(self, cmd): if isinstance(cmd, RemotePath): if cmd.remote is self: return self.RemoteCommand(self, cmd) - else: - raise TypeError( - "Given path does not belong to this remote machine: {!r}".format( - cmd - ) - ) - elif not isinstance(cmd, LocalPath): - if "/" in cmd or "\\" in cmd: - return self.RemoteCommand(self, self.path(cmd)) - else: - return self.RemoteCommand(self, self.which(cmd)) - else: - raise TypeError(f"cmd must not be a LocalPath: {cmd!r}") + + raise TypeError( + f"Given path does not belong to this remote machine: {cmd!r}" + ) + + if not isinstance(cmd, LocalPath): + return self.RemoteCommand( + self, self.path(cmd) if "/" in cmd or "\\" in cmd else self.which(cmd) + ) + + raise TypeError(f"cmd must not be a LocalPath: {cmd!r}") @property def python(self): @@ -322,11 +321,11 @@ def tempdir(self): _, out, _ = self._session.run( "mktemp -d 2>/dev/null || mktemp -d tmp.XXXXXXXXXX" ) - dir = self.path(out.strip()) # @ReservedAssignment + local_dir = self.path(out.strip()) try: - yield dir + yield local_dir finally: - dir.delete() + local_dir.delete() # # Path implementation @@ -387,7 +386,12 @@ def _path_move(self, src, dst): def _path_copy(self, src, dst): self._session.run(f"cp -r {shquote(src)} {shquote(dst)}") - def _path_mkdir(self, fn, mode=None, minus_p=True): + def _path_mkdir( + self, + fn, + mode=None, # pylint: disable=unused-argument + minus_p=True, + ): p_str = "-p " if minus_p else "" cmd = f"mkdir {p_str}{shquote(fn)}" self._session.run(cmd) @@ -427,9 +431,8 @@ def _path_write(self, fn, data): self.upload(f.name, fn) def _path_link(self, src, dst, symlink): - self._session.run( - "ln {} {} {}".format("-s" if symlink else "", shquote(src), shquote(dst)) - ) + symlink_str = "-s " if symlink else "" + self._session.run(f"ln {symlink_str}{shquote(src)} {shquote(dst)}") def expand(self, expr): return self._session.run(f"echo {expr}")[1].strip() @@ -438,4 +441,5 @@ def expanduser(self, expr): if not any(part.startswith("~") for part in expr.split("/")): return expr # we escape all $ signs to avoid expanding env-vars - return self._session.run("echo {}".format(expr.replace("$", "\\$")))[1].strip() + expr_repl = expr.replace("$", "\\$") + return self._session.run(f"echo {expr_repl}")[1].strip() diff --git a/plumbum/machines/session.py b/plumbum/machines/session.py index d9ba6a274..21d132176 100644 --- a/plumbum/machines/session.py +++ b/plumbum/machines/session.py @@ -12,8 +12,6 @@ class ShellSessionError(Exception): """Raises when something goes wrong when calling :func:`ShellSession.popen `""" - pass - class SSHCommsError(ProcessExecutionError, EOFError): """Raises when the communication channel can't be created on the @@ -87,17 +85,14 @@ def __init__(self, proc, argv, isatty, stdin, stdout, stderr, encoding): def poll(self): """Returns the process' exit code or ``None`` if it's still running""" - if self._done: - return self.returncode - else: - return None + return self.returncode if self._done else None def wait(self): """Waits for the process to terminate and returns its exit code""" self.communicate() return self.returncode - def communicate(self, input=None): + def communicate(self, input=None): # pylint: disable=redefined-builtin """Consumes the process' stdout and stderr until the it terminates. :param input: An optional bytes/buffer object to send to the process over stdin @@ -121,7 +116,7 @@ def communicate(self, input=None): try: line = pipe.readline() shell_logger.debug("%s> %r", name, line) - except EOFError: + except EOFError as err: shell_logger.debug("%s> Nothing returned.", name) self.proc.poll() @@ -137,39 +132,39 @@ def communicate(self, input=None): stdout, stderr, message="Incorrect username or password provided", - ) - elif returncode == 6: + ) from None + if returncode == 6: raise HostPublicKeyUnknown( argv, returncode, stdout, stderr, message="The authenticity of the host can't be established", - ) - elif returncode != 0: + ) from None + if returncode != 0: raise SSHCommsError( argv, returncode, stdout, stderr, message="SSH communication failed", - ) - elif name == "2": + ) from None + if name == "2": raise SSHCommsChannel2Error( argv, returncode, stdout, stderr, message="No stderr result detected. Does the remote have Bash as the default shell?", - ) - else: - raise SSHCommsError( - argv, - returncode, - stdout, - stderr, - message="No communication channel detected. Does the remote exist?", - ) + ) from None + + raise SSHCommsError( + argv, + returncode, + stdout, + stderr, + message="No communication channel detected. Does the remote exist?", + ) from err if not line: del sources[i] else: @@ -221,7 +216,7 @@ def closer(): ) self.close() - timer = threading.Timer(connect_timeout, self.close) + timer = threading.Timer(connect_timeout, closer) timer.start() try: self.run("") diff --git a/plumbum/machines/ssh_machine.py b/plumbum/machines/ssh_machine.py index be1f3dc83..8ce41c437 100644 --- a/plumbum/machines/ssh_machine.py +++ b/plumbum/machines/ssh_machine.py @@ -19,10 +19,8 @@ def __init__(self, session): self._session = session def __repr__(self): - if self._session.alive(): - return f"" - else: - return "" + tunnel = self._session.proc if self._session.alive() else "(defunct)" + return f"" def __enter__(self): return self @@ -218,7 +216,7 @@ def tunnel( dport, lhost="localhost", dhost="localhost", - connect_timeout=5, + connect_timeout=5, # pylint: disable=unused-argument reverse=False, ): r"""Creates an SSH tunnel from the TCP port (``lport``) of the local machine @@ -290,7 +288,7 @@ def tunnel( ) ) - def _translate_drive_letter(self, path): + def _translate_drive_letter(self, path): # pylint: disable=no-self-use # replace c:\some\path with /c/some/path path = str(path) if ":" in path: diff --git a/plumbum/path/base.py b/plumbum/path/base.py index d45906b5e..a1a93aaea 100644 --- a/plumbum/path/base.py +++ b/plumbum/path/base.py @@ -2,7 +2,7 @@ import operator import os import warnings -from abc import ABC, abstractmethod, abstractproperty +from abc import ABC, abstractmethod from functools import reduce FLAGS = {"f": os.F_OK, "w": os.W_OK, "r": os.R_OK, "x": os.X_OK} @@ -51,16 +51,16 @@ def __iter__(self): def __eq__(self, other): if isinstance(other, Path): return self._get_info() == other._get_info() - elif isinstance(other, str): + if isinstance(other, str): if self.CASE_SENSITIVE: return str(self) == other - else: - return str(self).lower() == other.lower() - else: - return NotImplemented + + return str(self).lower() == other.lower() + + return NotImplemented def __ne__(self, other): - return not (self == other) + return not self == other def __gt__(self, other): return str(self) > str(other) @@ -75,10 +75,7 @@ def __le__(self, other): return str(self) <= str(other) def __hash__(self): - if self.CASE_SENSITIVE: - return hash(str(self)) - else: - return hash(str(self).lower()) + return hash(str(self)) if self.CASE_SENSITIVE else hash(str(self).lower()) def __bool__(self): return bool(str(self)) @@ -103,8 +100,10 @@ def up(self, count=1): return self.join("../" * count) def walk( - self, filter=lambda p: True, dir_filter=lambda p: True - ): # @ReservedAssignment + self, + filter=lambda p: True, # pylint: disable=redefined-builtin + dir_filter=lambda p: True, + ): """traverse all (recursive) sub-elements under this directory, that match the given filter. By default, the filter accepts everything; you can provide a custom filter function that takes a path as an argument and returns a boolean @@ -120,7 +119,8 @@ def walk( if p.is_dir() and dir_filter(p): yield from p.walk(filter, dir_filter) - @abstractproperty + @property + @abstractmethod def name(self): """The basename component of this path""" @@ -130,37 +130,45 @@ def basename(self): warnings.warn("Use .name instead", FutureWarning) return self.name - @abstractproperty + @property + @abstractmethod def stem(self): """The name without an extension, or the last component of the path""" - @abstractproperty + @property + @abstractmethod def dirname(self): """The dirname component of this path""" - @abstractproperty + @property + @abstractmethod def root(self): """The root of the file tree (`/` on Unix)""" - @abstractproperty + @property + @abstractmethod def drive(self): """The drive letter (on Windows)""" - @abstractproperty + @property + @abstractmethod def suffix(self): """The suffix of this file""" - @abstractproperty + @property + @abstractmethod def suffixes(self): """This is a list of all suffixes""" - @abstractproperty + @property + @abstractmethod def uid(self): """The user that owns this path. The returned value is a :class:`FSUser ` object which behaves like an ``int`` (as expected from ``uid``), but it also has a ``.name`` attribute that holds the string-name of the user""" - @abstractproperty + @property + @abstractmethod def gid(self): """The group that owns this path. The returned value is a :class:`FSUser ` object which behaves like an ``int`` (as expected from ``gid``), but it also has a ``.name`` @@ -220,7 +228,6 @@ def exists(self): @abstractmethod def stat(self): """Returns the os.stats for a file""" - pass @abstractmethod def with_name(self, name): @@ -236,10 +243,7 @@ def with_suffix(self, suffix, depth=1): def preferred_suffix(self, suffix): """Adds a suffix if one does not currently exist (otherwise, no change). Useful for loading files with a default suffix""" - if len(self.suffixes) > 0: - return self - else: - return self.with_suffix(suffix) + return self if len(self.suffixes) > 0 else self.with_suffix(suffix) @abstractmethod def glob(self, pattern): @@ -285,7 +289,7 @@ def mkdir(self, mode=0o777, parents=True, exist_ok=True): """ @abstractmethod - def open(self, mode="r"): + def open(self, mode="r", *, encoding=None): """opens this path as a file""" @abstractmethod @@ -322,9 +326,13 @@ def chmod(self, mode): """ @staticmethod - def _access_mode_to_flags(mode, flags=FLAGS): + def _access_mode_to_flags(mode, flags=None): + if flags is None: + flags = FLAGS + if isinstance(mode, str): mode = reduce(operator.or_, [flags[m] for m in mode.lower()], 0) + return mode @abstractmethod @@ -354,7 +362,7 @@ def symlink(self, dst): def unlink(self): """Deletes a symbolic link""" - def split(self, *dummy_args, **dummy_kargs): + def split(self, *_args, **_kargs): """Splits the path on directory separators, yielding a list of directories, e.g, ``"/var/log/messages"`` will yield ``['var', 'log', 'messages']``. """ @@ -394,17 +402,18 @@ def __sub__(self, other): """Same as ``self.relative_to(other)``""" return self.relative_to(other) - def _glob(self, pattern, fn): + @staticmethod + def _glob(pattern, fn): """Applies a glob string or list/tuple/iterable to the current path, using ``fn``""" if isinstance(pattern, str): return fn(pattern) - else: - results = [] - for single_pattern in pattern: - results.extend(fn(single_pattern)) - return sorted(list(set(results))) - def resolve(self, strict=False): + results = [] + for single_pattern in pattern: + results.extend(fn(single_pattern)) + return sorted(list(set(results))) + + def resolve(self, strict=False): # pylint:disable=unused-argument """Added to allow pathlib like syntax. Does nothing since Plumbum paths are always absolute. Does not (currently) resolve symlinks.""" @@ -459,7 +468,7 @@ def __eq__(self, other): return str(self) == str(other) def __ne__(self, other): - return not (self == other) + return not self == other def __gt__(self, other): return str(self) > str(other) diff --git a/plumbum/path/local.py b/plumbum/path/local.py index b4080fe2a..b8a633267 100644 --- a/plumbum/path/local.py +++ b/plumbum/path/local.py @@ -4,6 +4,8 @@ import os import shutil import sys +import urllib.parse as urlparse +import urllib.request as urllib from contextlib import contextmanager from plumbum.lib import IS_WIN32 @@ -15,24 +17,23 @@ from pwd import getpwnam, getpwuid except ImportError: - def getpwuid(x): # type: ignore + def getpwuid(_x): # type: ignore return (None,) - def getgrgid(x): # type: ignore + def getgrgid(_x): # type: ignore return (None,) - def getpwnam(x): # type: ignore + def getpwnam(_x): # type: ignore raise OSError("`getpwnam` not supported") - def getgrnam(x): # type: ignore + def getgrnam(_x): # type: ignore raise OSError("`getgrnam` not supported") -import urllib.parse as urlparse -import urllib.request as urllib - logger = logging.getLogger("plumbum.local") +_EMPTY = object() + # =================================================================================================== # Local Paths @@ -139,7 +140,7 @@ def stem(self): def with_suffix(self, suffix, depth=1): if suffix and not suffix.startswith(os.path.extsep) or suffix == os.path.extsep: - raise ValueError("Invalid suffix %r" % (suffix)) + raise ValueError(f"Invalid suffix {suffix!r}") name = self.name depth = len(self.suffixes) if depth is None else min(depth, len(self.suffixes)) for _ in range(depth): @@ -202,8 +203,12 @@ def mkdir(self, mode=0o777, parents=True, exist_ok=True): if ex.errno != errno.EEXIST or not exist_ok: raise - def open(self, mode="r"): - return open(str(self), mode) + def open(self, mode="r", encoding=None): + return open( + str(self), + mode, + encoding=encoding, + ) def read(self, encoding=None, mode="r"): if encoding and "b" not in mode: @@ -226,7 +231,7 @@ def write(self, data, encoding=None, mode=None): f.write(data) def touch(self): - with open(str(self), "a"): + with open(str(self), "a", encoding="utf-8"): os.utime(str(self), None) def chown(self, owner=None, group=None, recursive=None): diff --git a/plumbum/path/remote.py b/plumbum/path/remote.py index b9afba68a..b957b64ba 100644 --- a/plumbum/path/remote.py +++ b/plumbum/path/remote.py @@ -59,7 +59,7 @@ def __new__(cls, remote, *parts): plist.pop(0) del normed[:] for item in plist: - if item == "" or item == ".": + if item in {"", "."}: continue if item == "..": if normed: @@ -166,7 +166,7 @@ def with_name(self, name): def with_suffix(self, suffix, depth=1): if suffix and not suffix.startswith(".") or suffix == ".": - raise ValueError("Invalid suffix %r" % (suffix)) + raise ValueError(f"Invalid suffix {suffix!r}") name = self.name depth = len(self.suffixes) if depth is None else min(depth, len(self.suffixes)) for _ in range(depth): @@ -192,8 +192,7 @@ def move(self, dst): raise TypeError("dst points to a different remote machine") elif not isinstance(dst, str): raise TypeError( - "dst must be a string or a RemotePath (to the same remote machine), " - "got %r" % (dst,) + f"dst must be a string or a RemotePath (to the same remote machine), got {dst!r}" ) self.remote._path_move(self, dst) @@ -203,8 +202,7 @@ def copy(self, dst, override=False): raise TypeError("dst points to a different remote machine") elif not isinstance(dst, str): raise TypeError( - "dst must be a string or a RemotePath (to the same remote machine), " - "got %r" % (dst,) + f"dst must be a string or a RemotePath (to the same remote machine), got {dst!r}" ) if override: if isinstance(dst, str): @@ -228,14 +226,14 @@ def mkdir(self, mode=None, parents=True, exist_ok=True): self.remote._path_mkdir(self, mode=mode, minus_p=False) except ProcessExecutionError: _, ex, _ = sys.exc_info() - if "File exists" in ex.stderr: - if not exist_ok: - raise OSError( - errno.EEXIST, "File exists (on remote end)", str(self) - ) - else: + if "File exists" not in ex.stderr: raise + if not exist_ok: + raise OSError( + errno.EEXIST, "File exists (on remote end)", str(self) + ) from None + def read(self, encoding=None): data = self.remote._path_read(self) if encoding: @@ -272,8 +270,7 @@ def link(self, dst): raise TypeError("dst points to a different remote machine") elif not isinstance(dst, str): raise TypeError( - "dst must be a string or a RemotePath (to the same remote machine), " - "got %r" % (dst,) + f"dst must be a string or a RemotePath (to the same remote machine), got {dst!r}" ) self.remote._path_link(self, dst, False) @@ -283,30 +280,32 @@ def symlink(self, dst): raise TypeError("dst points to a different remote machine") elif not isinstance(dst, str): raise TypeError( - "dst must be a string or a RemotePath (to the same remote machine), " - "got %r" % (dst,) + "dst must be a string or a RemotePath (to the same remote machine), got {dst!r}" ) self.remote._path_link(self, dst, True) - def open(self, mode="r", bufsize=-1): + def open(self, mode="r", bufsize=-1, *, encoding=None): """ Opens this path as a file. Only works for ParamikoMachine-associated paths for now. """ - if hasattr(self.remote, "sftp") and hasattr(self.remote.sftp, "open"): - return self.remote.sftp.open(self, mode, bufsize) - else: + if encoding is not None: raise NotImplementedError( - "RemotePath.open only works for ParamikoMachine-associated " - "paths for now" + "encoding not supported for ParamikoMachine paths" ) - def as_uri(self, scheme="ssh"): - return "{}://{}{}".format( - scheme, self.remote._fqhost, urllib.pathname2url(str(self)) + if hasattr(self.remote, "sftp") and hasattr(self.remote.sftp, "open"): + return self.remote.sftp.open(self, mode, bufsize) + + raise NotImplementedError( + "RemotePath.open only works for ParamikoMachine-associated paths for now" ) + def as_uri(self, scheme="ssh"): + suffix = urllib.pathname2url(str(self)) + return f"{scheme}://{self.remote._fqhost}{suffix}" + @property def stem(self): return self.name.rsplit(".")[0] diff --git a/plumbum/path/utils.py b/plumbum/path/utils.py index 9b64afc7d..558b93114 100644 --- a/plumbum/path/utils.py +++ b/plumbum/path/utils.py @@ -48,20 +48,17 @@ def move(src, dst): for src2 in src: move(src2, dst) return dst - elif not isinstance(src, Path): + if not isinstance(src, Path): src = local.path(src) if isinstance(src, LocalPath): - if isinstance(dst, LocalPath): - return src.move(dst) - else: - return _move(src, dst) - elif isinstance(dst, LocalPath): + return src.move(dst) if isinstance(dst, LocalPath) else _move(src, dst) + if isinstance(dst, LocalPath): return _move(src, dst) - elif src.remote == dst.remote: + if src.remote == dst.remote: return src.move(dst) - else: - return _move(src, dst) + + return _move(src, dst) def copy(src, dst): @@ -86,25 +83,27 @@ def copy(src, dst): for src2 in src: copy(src2, dst) return dst - elif not isinstance(src, Path): + + if not isinstance(src, Path): src = local.path(src) if isinstance(src, LocalPath): if isinstance(dst, LocalPath): return src.copy(dst) - else: - dst.remote.upload(src, dst) - return dst - elif isinstance(dst, LocalPath): + dst.remote.upload(src, dst) + return dst + + if isinstance(dst, LocalPath): src.remote.download(src, dst) return dst - elif src.remote == dst.remote: + + if src.remote == dst.remote: return src.copy(dst) - else: - with local.tempdir() as tmp: - copy(src, tmp) - copy(tmp / src.name, dst) - return dst + + with local.tempdir() as tmp: + copy(src, tmp) + copy(tmp / src.name, dst) + return dst def gui_open(filename): diff --git a/plumbum/typed_env.py b/plumbum/typed_env.py index 3042ee294..4ad15d501 100644 --- a/plumbum/typed_env.py +++ b/plumbum/typed_env.py @@ -47,7 +47,7 @@ def __init__(self, name, default=NO_DEFAULT): self.name = self.names[0] self.default = default - def convert(self, value): + def convert(self, value): # pylint:disable=no-self-use return value def __get__(self, instance, owner): @@ -72,11 +72,11 @@ class Bool(_BaseVar): Case-insensitive. Throws a ``ValueError`` for any other value. """ - def convert(self, s): - s = s.lower() - if s not in ("yes", "no", "true", "false", "1", "0"): - raise ValueError(f"Unrecognized boolean value: {s!r}") - return s in ("yes", "true", "1") + def convert(self, value): + value = value.lower() + if value not in {"yes", "no", "true", "false", "1", "0"}: + raise ValueError(f"Unrecognized boolean value: {value!r}") + return value in {"yes", "true", "1"} def __set__(self, instance, value): instance[self.name] = "yes" if value else "no" @@ -93,7 +93,9 @@ class CSV(_BaseVar): a list of objects of type ``type`` (``str`` by default). """ - def __init__(self, name, default=NO_DEFAULT, type=str, separator=","): + def __init__( + self, name, default=NO_DEFAULT, type=str, separator="," + ): # pylint:disable=redefined-builtin super(TypedEnv.CSV, self).__init__(name, default=default) self.type = type self.separator = separator @@ -106,7 +108,9 @@ def convert(self, value): # ========= - def __init__(self, env=os.environ): + def __init__(self, env=None): + if env is None: + env = os.environ self._env = env self._defined_keys = { k @@ -131,8 +135,7 @@ def _raw_get(self, *key_names): value = self._env.get(key, NO_DEFAULT) if value is not NO_DEFAULT: return value - else: - raise EnvironmentVariableError(key_names[0]) + raise EnvironmentVariableError(key_names[0]) def __contains__(self, key): try: @@ -147,7 +150,9 @@ def __getattr__(self, name): try: return self._raw_get(name) except EnvironmentVariableError: - raise AttributeError(f"{self.__class__} has no attribute {name!r}") + raise AttributeError( + f"{self.__class__} has no attribute {name!r}" + ) from None def __getitem__(self, key): return getattr(self, key) # delegate through the descriptors diff --git a/pyproject.toml b/pyproject.toml index e75233134..b67de392b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,44 @@ ignore = [ [tool.pycln] all = true + + +[tool.pylint] +master.py-version = "3.6" +master.jobs = "0" +master.fail-under = "9.97" +reports.output-format = "colorized" +similarities.ignore-imports = "yes" +messages_control.enable = [ + "useless-suppression", +] +messages_control.disable = [ + "arguments-differ", # TODO: investigate + "attribute-defined-outside-init", # TODO: investigate + "broad-except", # TODO: investigate + "cyclic-import", + "duplicate-code", # TODO: check + "fixme", + "import-error", + "import-outside-toplevel", # TODO: see if this can be limited to certain imports + "invalid-name", + "line-too-long", + "missing-class-docstring", + "missing-function-docstring", + "missing-module-docstring", + "no-member", + "protected-access", + "too-few-public-methods", + "too-many-arguments", + "too-many-branches", + "too-many-function-args", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-nested-blocks", + "too-many-public-methods", + "too-many-return-statements", + "too-many-statements", + "non-parent-init-called", # TODO: should be looked at + "unidiomatic-typecheck", # TODO: might be able to remove +] From 9d070ac7ba10eec66702842c20974db884a24005 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 28 Jan 2022 10:32:10 -0500 Subject: [PATCH 06/37] fix: add types and fix issue with ioctrl + Python 3 (#579) --- plumbum/cli/i18n.py | 2 +- plumbum/cli/termsize.py | 10 ++++++---- plumbum/colorlib/names.py | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/plumbum/cli/i18n.py b/plumbum/cli/i18n.py index 4bcd6616b..915231a47 100644 --- a/plumbum/cli/i18n.py +++ b/plumbum/cli/i18n.py @@ -5,7 +5,7 @@ if loc is None or loc.startswith("en"): class NullTranslation: - def gettext(self, str1): # pylint: disable=no-self-use + def gettext(self, str1: str) -> str: # pylint: disable=no-self-use return str1 def ngettext(self, str1, strN, n): # pylint: disable=no-self-use diff --git a/plumbum/cli/termsize.py b/plumbum/cli/termsize.py index 8bc23f3fb..d0341b86e 100644 --- a/plumbum/cli/termsize.py +++ b/plumbum/cli/termsize.py @@ -7,11 +7,12 @@ import platform import warnings from struct import Struct +from typing import Optional, Tuple from plumbum import local -def get_terminal_size(default=(80, 25)): +def get_terminal_size(default: Tuple[int, int] = (80, 25)) -> Tuple[int, int]: """ Get width and height of console; works on linux, os x, windows and cygwin @@ -71,18 +72,19 @@ def _get_terminal_size_tput(): # pragma: no cover return None -def _ioctl_GWINSZ(fd): +def _ioctl_GWINSZ(fd: int) -> Optional[Tuple[int, int]]: yx = Struct("hh") try: import fcntl import termios - return yx.unpack(fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234")) + # TODO: Clean this up. Problems could be hidden by the broad except. + return yx.unpack(fcntl.ioctl(fd, termios.TIOCGWINSZ, b"1234")) # type: ignore[return-value] except Exception: return None -def _get_terminal_size_linux(): +def _get_terminal_size_linux() -> Optional[Tuple[int, int]]: cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) if not cr: try: diff --git a/plumbum/colorlib/names.py b/plumbum/colorlib/names.py index ca5a7b7c3..612885579 100644 --- a/plumbum/colorlib/names.py +++ b/plumbum/colorlib/names.py @@ -5,7 +5,7 @@ You can access the index of the colors with names.index(name). You can access the rgb values with ``r=int(html[n][1:3],16)``, etc. """ - +from typing import Tuple color_names = """\ black @@ -345,7 +345,7 @@ class FindNearest: """This is a class for finding the nearest color given rgb values. Different find methods are available.""" - def __init__(self, r, g, b): + def __init__(self, r: int, g: int, b: int) -> None: self.r = r self.b = b self.g = g @@ -366,23 +366,23 @@ def only_basic(self): + (self.b >= midlevel) * 4 ) - def all_slow(self, color_slice=EMPTY_SLICE): + def all_slow(self, color_slice: slice = EMPTY_SLICE) -> int: """This is a slow way to find the nearest color.""" distances = [ self._distance_to_color(color) for color in color_html[color_slice] ] return min(range(len(distances)), key=distances.__getitem__) - def _distance_to_color(self, color): + def _distance_to_color(self, color: str) -> int: """This computes the distance to a color, should be minimized.""" rgb = (int(color[1:3], 16), int(color[3:5], 16), int(color[5:7], 16)) return (self.r - rgb[0]) ** 2 + (self.g - rgb[1]) ** 2 + (self.b - rgb[2]) ** 2 - def _distance_to_color_number(self, n): + def _distance_to_color_number(self, n: int) -> int: color = color_html[n] return self._distance_to_color(color) - def only_colorblock(self): + def only_colorblock(self) -> int: """This finds the nearest color based on block system, only works for 17-232 color values.""" rint = min( @@ -396,11 +396,11 @@ def only_colorblock(self): ) return 16 + 36 * rint + 6 * gint + bint - def only_simple(self): + def only_simple(self) -> int: """Finds the simple color-block color.""" return self.all_slow(slice(0, 16, None)) - def only_grey(self): + def only_grey(self) -> int: """Finds the greyscale color.""" rawval = (self.r + self.b + self.g) / 3 n = min( @@ -409,14 +409,14 @@ def only_grey(self): ) return n + 232 - def all_fast(self): + def all_fast(self) -> int: """Runs roughly 8 times faster than the slow version.""" colors = [self.only_simple(), self.only_colorblock(), self.only_grey()] distances = [self._distance_to_color_number(n) for n in colors] return colors[min(range(len(distances)), key=distances.__getitem__)] -def from_html(color): +def from_html(color: str) -> Tuple[int, int, int]: """Convert html hex code to rgb.""" if len(color) != 7 or color[0] != "#": raise ValueError("Invalid length of html code") From b71a852a4b049bdc4aa2cfedb43dad395d393ef2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 28 Jan 2022 12:43:27 -0500 Subject: [PATCH 07/37] chore: restrict skips and check rst (#580) --- .pre-commit-config.yaml | 12 ++++++++++++ CHANGELOG.rst | 6 +++--- docs/cli.rst | 18 ++++++++++++------ docs/colors.rst | 2 +- docs/local_commands.rst | 2 +- docs/utils.rst | 2 +- plumbum/__init__.py | 2 +- plumbum/colorlib/styles.py | 7 ++++--- plumbum/commands/base.py | 9 +++++---- plumbum/machines/paramiko_machine.py | 4 ++-- plumbum/path/local.py | 8 ++++---- pyproject.toml | 4 +++- 12 files changed, 49 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1cd9077c2..f3648599c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,3 +73,15 @@ repos: rev: v2.1.0 hooks: - id: codespell + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: "v1.9.0" + hooks: + - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore + - id: python-no-log-warn + - id: python-no-eval + - id: python-use-type-annotations + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 99ddc27db..760613253 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -58,7 +58,7 @@ * Paths: support composing paths using subscription operations (`#455 `_) * CLI: Improved 'Set' validator to allow non-string types, and CSV params (`#452 `_) * TypedEnv: Facility for modeling environment-variables into python data types (`#451 `_) -* Commands: execute local/remote commands via a magic `.cmd` attribute (`#450 `_) +* Commands: execute local/remote commands via a magic ``.cmd`` attribute (`#450 `_) 1.6.7 ----- @@ -138,7 +138,7 @@ * Bugfix: Getting an executable no longer returns a directory (`#234 `_) * Bugfix: Iterdir now works on Python <3.5 * Testing is now expanded and fully written in Pytest, with coverage reporting. -* Added support for Conda ( as of 1.6.2, use the `-c conda-forge` channel) +* Added support for Conda ( as of 1.6.2, use the ``-c conda-forge`` channel) 1.6.0 ----- @@ -164,7 +164,7 @@ * CLI: add ``--help-all`` and various cosmetic fixes: (`#125 `_), (`#126 `_), (`#127 `_) * CLI: add ``root_app`` property (`#141 `_) -* Machines: ``getattr`` now raises ``AttributeError`` instead of `CommandNotFound` (`#135 `_) +* Machines: ``getattr`` now raises ``AttributeError`` instead of ``CommandNotFound`` (`#135 `_) * Paramiko: ``keep_alive`` support (`#186 `_) * Paramiko: does not support piping explicitly now (`#160 `_) * Parmaiko: Added pure SFTP backend, gives STFP v4+ support (`#188 `_) diff --git a/docs/cli.rst b/docs/cli.rst index 7b45ed328..3a6b9d4c6 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -219,9 +219,9 @@ these values. Here's an example :: ValueError("Expected one of ['UDP', 'TCP']",) .. note:: - The toolkit also provides some other useful validators: `ExistingFile` (ensures the given - argument is an existing file), `ExistingDirectory` (ensures the given argument is an existing - directory), and `NonexistentPath` (ensures the given argument is not an existing path). + The toolkit also provides some other useful validators: ``ExistingFile`` (ensures the given + argument is an existing file), ``ExistingDirectory`` (ensures the given argument is an existing + directory), and ``NonexistentPath`` (ensures the given argument is not an existing path). All of these convert the argument to a :ref:`local path `. @@ -598,9 +598,15 @@ A command line image plotter (``Image``) is provided in ``plumbum.cli.image``. I Image().show_pil(im) -The Image constructor can take an optional size (defaults to the current terminal size if None), and a `char_ratio`, a height to width measure for your current font. It defaults to a common value of 2.45. If set to None, the ratio is ignored and the image will no longer be constrained to scale proportionately. To directly plot an image, the ``show`` method takes a filename and a double parameter, which doubles the vertical resolution on some fonts. The `show_pil` and `show_pil_double` -methods directly take a PIL-like object. To plot an image from the command line, -the module can be run directly: ``python -m plumbum.cli.image myimage.png``. +The Image constructor can take an optional size (defaults to the current +terminal size if None), and a ``char_ratio``, a height to width measure for your +current font. It defaults to a common value of 2.45. If set to None, the ratio +is ignored and the image will no longer be constrained to scale proportionately. +To directly plot an image, the ``show`` method takes a filename and a double +parameter, which doubles the vertical resolution on some fonts. The ``show_pil`` +and ``show_pil_double`` methods directly take a PIL-like object. To plot an image +from the command line, the module can be run directly: ``python -m +plumbum.cli.image myimage.png``. For the full list of helpers or more information, see the :ref:`api docs `. diff --git a/docs/colors.rst b/docs/colors.rst index f8bd1eac2..36543e94b 100644 --- a/docs/colors.rst +++ b/docs/colors.rst @@ -6,7 +6,7 @@ Colors .. versionadded:: 1.6 -The purpose of the `plumbum.colors` library is to make adding +The purpose of the ``plumbum.colors`` library is to make adding text styles (such as color) to Python easy and safe. Color is often a great addition to shell scripts, but not a necessity, and implementing it properly is tricky. It is easy to end up with an unreadable color stuck on your terminal or diff --git a/docs/local_commands.rst b/docs/local_commands.rst index aea58c3af..9e1d0c724 100644 --- a/docs/local_commands.rst +++ b/docs/local_commands.rst @@ -164,7 +164,7 @@ one you passed:: For instance, ``grep("foo", "myfile.txt", retcode = (0, 2))`` If you need to have both the output/error and the exit code (using exceptions would provide either - but not both), you can use the `run` method, which will provide all of them + but not both), you can use the ``run`` method, which will provide all of them >>> cat["non/existing.file"].run(retcode=None) (1, u'', u'/bin/cat: non/existing.file: No such file or directory\n') diff --git a/docs/utils.rst b/docs/utils.rst index 9320d2d03..682998fd6 100644 --- a/docs/utils.rst +++ b/docs/utils.rst @@ -31,4 +31,4 @@ imported into the namespace of ``plumbum`` directly, and you have to explicitly from plumbum.path.utils import delete delete(local.cwd // "*/*.pyc", local.cwd // "*/__pycache__") -* :func:`gui_open(path) ` - Opens a file in the default editor on Windows, Mac, or Linux. Uses `os.startfile` if available (Windows), `xdg_open` (GNU), or `open` (Mac). +* :func:`gui_open(path) ` - Opens a file in the default editor on Windows, Mac, or Linux. Uses ``os.startfile`` if available (Windows), ``xdg_open`` (GNU), or ``open`` (Mac). diff --git a/plumbum/__init__.py b/plumbum/__init__.py index a0098a222..4795495c1 100644 --- a/plumbum/__init__.py +++ b/plumbum/__init__.py @@ -103,7 +103,7 @@ def __getattr__(self, name): except CommandNotFound: raise AttributeError(name) from None - __path__ = [] # type: List[str] + __path__: List[str] = [] __file__ = __file__ diff --git a/plumbum/colorlib/styles.py b/plumbum/colorlib/styles.py index 38db87313..eed9838e5 100644 --- a/plumbum/colorlib/styles.py +++ b/plumbum/colorlib/styles.py @@ -14,7 +14,7 @@ import sys from abc import ABC, abstractmethod from copy import copy -from typing import IO, Dict, Union +from typing import IO, Dict, Optional, Union from .names import ( FindNearest, @@ -343,8 +343,9 @@ class Style: """The class of color to use. Never hardcode ``Color`` call when writing a Style method.""" - attribute_names = None # type: Union[Dict[str,str], Dict[str,int]] - _stdout = None # type: IO + attribute_names: Union[Dict[str, str], Dict[str, int]] + + _stdout: Optional[IO] = None end = "\n" """The endline character. Override if needed in subclasses.""" diff --git a/plumbum/commands/base.py b/plumbum/commands/base.py index ec199c573..c15c3ba7f 100644 --- a/plumbum/commands/base.py +++ b/plumbum/commands/base.py @@ -5,6 +5,7 @@ from subprocess import PIPE, Popen from tempfile import TemporaryFile from types import MethodType +from typing import ClassVar import plumbum.commands.modifiers from plumbum.commands.processes import iter_lines, run_proc @@ -440,9 +441,9 @@ def verify(proc, retcode, timeout, stdout, stderr): class BaseRedirection(BaseCommand): __slots__ = ("cmd", "file") - SYM = None # type: str - KWARG = None # type: str - MODE = "r" # type: str + SYM: ClassVar[str] + KWARG: ClassVar[str] + MODE: ClassVar[str] def __init__(self, cmd, file): self.cmd = cmd @@ -564,7 +565,7 @@ def popen(self, args=(), **kwargs): class ConcreteCommand(BaseCommand): - QUOTE_LEVEL = None # type: int + QUOTE_LEVEL: ClassVar[int] __slots__ = ("executable", "custom_encoding") def __init__(self, executable, encoding): diff --git a/plumbum/machines/paramiko_machine.py b/plumbum/machines/paramiko_machine.py index 3a5858e00..1c5559eea 100644 --- a/plumbum/machines/paramiko_machine.py +++ b/plumbum/machines/paramiko_machine.py @@ -16,14 +16,14 @@ import paramiko except ImportError: - class paramiko: # type: ignore + class paramiko: # type: ignore[no-redef] def __bool__(self): return False def __getattr__(self, name): raise ImportError("No module named paramiko") - paramiko = paramiko() # type: ignore + paramiko = paramiko() # type: ignore[operator] logger = logging.getLogger("plumbum.paramiko") diff --git a/plumbum/path/local.py b/plumbum/path/local.py index b8a633267..38edf5ba0 100644 --- a/plumbum/path/local.py +++ b/plumbum/path/local.py @@ -17,16 +17,16 @@ from pwd import getpwnam, getpwuid except ImportError: - def getpwuid(_x): # type: ignore + def getpwuid(_x): # type: ignore[misc] return (None,) - def getgrgid(_x): # type: ignore + def getgrgid(_x): # type: ignore[misc] return (None,) - def getpwnam(_x): # type: ignore + def getpwnam(_x): # type: ignore[misc] raise OSError("`getpwnam` not supported") - def getgrnam(_x): # type: ignore + def getgrnam(_x): # type: ignore[misc] raise OSError("`getgrnam` not supported") diff --git a/pyproject.toml b/pyproject.toml index b67de392b..7f03dc928 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,9 @@ ignore_missing_imports = true minversion = "6.0" addopts = ["-ra", "--cov-config=setup.cfg", "--strict-markers", "--strict-config"] norecursedirs = ["examples", "experiments"] -# filterwarnings = ["error"] +filterwarnings = [ + "always" +] log_cli_level = "info" required_plugins = ["pytest-timeout", "pytest-mock"] timeout = 300 From 5e27bb109021c6718d8ff40ad85ec835418e9beb Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 28 Jan 2022 14:44:25 -0500 Subject: [PATCH 08/37] refactor: use Python 3.7 simpler impl for cmd (#581) --- plumbum/__init__.py | 33 ++++++++++++++++++--------------- plumbum/cmd.py | 9 +++++++++ 2 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 plumbum/cmd.py diff --git a/plumbum/__init__.py b/plumbum/__init__.py index 4795495c1..47f53c699 100644 --- a/plumbum/__init__.py +++ b/plumbum/__init__.py @@ -36,8 +36,6 @@ """ import sys -from types import ModuleType -from typing import List # Avoids a circular import error later import plumbum.path # noqa: F401 @@ -91,24 +89,29 @@ # =================================================================================================== -class LocalModule(ModuleType): - """The module-hack that allows us to use ``from plumbum.cmd import some_program``""" +if sys.version_info < (3, 7): + from types import ModuleType + from typing import List - __all__ = () # to make help() happy - __package__ = __name__ + class LocalModule(ModuleType): + """The module-hack that allows us to use ``from plumbum.cmd import some_program``""" - def __getattr__(self, name): - try: - return local[name] - except CommandNotFound: - raise AttributeError(name) from None + __all__ = () # to make help() happy + __package__ = __name__ - __path__: List[str] = [] - __file__ = __file__ + def __getattr__(self, name): + try: + return local[name] + except CommandNotFound: + raise AttributeError(name) from None + __path__: List[str] = [] + __file__ = __file__ -cmd = LocalModule(__name__ + ".cmd", LocalModule.__doc__) -sys.modules[cmd.__name__] = cmd + cmd = LocalModule(__name__ + ".cmd", LocalModule.__doc__) + sys.modules[cmd.__name__] = cmd +else: + from . import cmd def __dir__(): diff --git a/plumbum/cmd.py b/plumbum/cmd.py new file mode 100644 index 000000000..aa170266a --- /dev/null +++ b/plumbum/cmd.py @@ -0,0 +1,9 @@ +import plumbum + + +def __getattr__(name: str) -> plumbum.machines.LocalCommand: + """The module-hack that allows us to use ``from plumbum.cmd import some_program``""" + try: + return plumbum.local[name] + except plumbum.CommandNotFound: + raise AttributeError(name) from None From 55a5058ccc20745bae77db7b29ec7f1c1f9daef5 Mon Sep 17 00:00:00 2001 From: Ofer Koren Date: Sat, 29 Jan 2022 21:48:06 +0200 Subject: [PATCH 09/37] feat(iter_lines): added new 'buffer_size' parameter, and updated docstrings (#582) --- plumbum/commands/processes.py | 25 +++++++++++++++++++++---- tests/test_local.py | 10 ++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 068b7a7d2..445d0611e 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -1,7 +1,6 @@ import atexit import heapq import time -from io import StringIO from queue import Empty as QueueEmpty from queue import Queue from threading import Thread @@ -304,6 +303,7 @@ def run_proc(proc, retcode, timeout=None): BY_POSITION = object() BY_TYPE = object() DEFAULT_ITER_LINES_MODE = BY_POSITION +DEFAULT_BUFFER_SIZE = _INFINITE = float("inf") def iter_lines( @@ -312,6 +312,7 @@ def iter_lines( timeout=None, linesize=-1, line_timeout=None, + buffer_size=None, mode=None, _iter_lines=_iter_lines, ): @@ -335,11 +336,23 @@ def iter_lines( Raise an :class:`ProcessLineTimedOut ` if the timeout has been reached. ``None`` means no timeout is imposed. + :param buffer_size: Maximum number of lines to keep in the stdout/stderr buffers, in case of a ProcessExecutionError. + Default is ``None``, which defaults to DEFAULT_BUFFER_SIZE (which is infinite by default). + ``0`` will disable bufferring completely. + + :param mode: Controls what the generator yields. Defaults to DEFAULT_ITER_LINES_MODE (which is BY_POSITION by default) + - BY_POSITION (default): yields ``(out, err)`` line tuples, where either item may be ``None`` + - BY_TYPE: yields ``(fd, line)`` tuples, where ``fd`` is 1 (stdout) or 2 (stderr) + :returns: An iterator of (out, err) line tuples. """ if mode is None: mode = DEFAULT_ITER_LINES_MODE + if buffer_size is None: + buffer_size = DEFAULT_BUFFER_SIZE + buffer_size: int + assert mode in (BY_POSITION, BY_TYPE) encoding = getattr(proc, "custom_encoding", None) or "utf-8" @@ -347,13 +360,17 @@ def iter_lines( _register_proc_timeout(proc, timeout) - buffers = [StringIO(), StringIO()] + buffers = [[], []] for t, line in _iter_lines(proc, decode, linesize, line_timeout): # verify that the proc hasn't timed out yet proc.verify(timeout=timeout, retcode=None, stdout=None, stderr=None) - buffers[t].write(line + "\n") + buffer = buffers[t] + if buffer_size > 0: + buffer.append(line) + if buffer_size < _INFINITE: + del buffer[:-buffer_size] if mode is BY_POSITION: ret = [None, None] @@ -363,4 +380,4 @@ def iter_lines( yield (t + 1), line # 1=stdout, 2=stderr # this will take care of checking return code and timeouts - _check_process(proc, retcode, timeout, *(s.getvalue() for s in buffers)) + _check_process(proc, retcode, timeout, *("\n".join(s) + "\n" for s in buffers)) diff --git a/tests/test_local.py b/tests/test_local.py index 43962eadf..27c9439aa 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -551,6 +551,16 @@ def test_iter_lines_timeout(self): print(i, "out:", out) assert i in (2, 3) # Mac is a bit flakey + @skip_on_windows + def test_iter_lines_buffer_size(self): + from plumbum.cmd import bash + + cmd = bash["-ce", "for ((i=0;i<100;i++)); do echo $i; done; false"] + with pytest.raises(ProcessExecutionError) as e: + for _ in cmd.popen().iter_lines(timeout=1, buffer_size=5): + pass + assert e.value.stdout == "\n".join(map(str, range(95, 100))) + "\n" + @skip_on_windows def test_iter_lines_timeout_by_type(self): from plumbum.cmd import bash From 32e314cb7749c5efaf57739dde8c701b4e5392a6 Mon Sep 17 00:00:00 2001 From: Ofer Koren Date: Wed, 2 Feb 2022 15:26:31 +0200 Subject: [PATCH 10/37] exceptions: fix ProcessExecutionError creating recursion errors --- plumbum/commands/processes.py | 2 +- tests/test_local.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 445d0611e..0b764cab0 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -114,7 +114,7 @@ class ProcessExecutionError(EnvironmentError): """ def __init__(self, argv, retcode, stdout, stderr, message=None): - super().__init__(self, argv, retcode, stdout, stderr) + super().__init__(argv, retcode, stdout, stderr) self.message = message self.argv = argv self.retcode = retcode diff --git a/tests/test_local.py b/tests/test_local.py index 27c9439aa..fedcd833e 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -724,6 +724,11 @@ def test_quoting(self): ls("-a", "") # check that empty strings are rendered correctly assert execinfo.value.argv[-2:] == ["-a", ""] + def test_exceptions(self): + with pytest.raises(ProcessExecutionError) as exc_info: + local.cmd.false() + assert repr(exc_info.value) == "ProcessExecutionError(['/usr/bin/false'], 1)" + def test_tempdir(self): with local.tempdir() as dir: assert dir.is_dir() From e635c16ff61f566b8b9e4ef5105d03a2ca6e04c5 Mon Sep 17 00:00:00 2001 From: Ofer Koren Date: Wed, 2 Feb 2022 16:14:03 +0200 Subject: [PATCH 11/37] exceptions: ensure ProcessExecutionError can be pickled --- plumbum/commands/processes.py | 6 +++++- tests/test_local.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 0b764cab0..f16b5a33c 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -114,7 +114,11 @@ class ProcessExecutionError(EnvironmentError): """ def __init__(self, argv, retcode, stdout, stderr, message=None): - super().__init__(argv, retcode, stdout, stderr) + + # we can't use 'super' here since EnvironmentError only keeps the first 2 args, + # which leads to failuring in loading this object from a pickle.dumps. + Exception.__init__(self, argv, retcode, stdout, stderr) + self.message = message self.argv = argv self.retcode = retcode diff --git a/tests/test_local.py b/tests/test_local.py index fedcd833e..448ce0b35 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -729,6 +729,12 @@ def test_exceptions(self): local.cmd.false() assert repr(exc_info.value) == "ProcessExecutionError(['/usr/bin/false'], 1)" + def test_exception_pickling(self): + import pickle + with pytest.raises(ProcessExecutionError) as exc_info: + local.cmd.ls("no-file") + assert pickle.loads(pickle.dumps(exc_info.value)).argv == exc_info.value.argv + def test_tempdir(self): with local.tempdir() as dir: assert dir.is_dir() From ca342091b512dee9d2ad4db36eb9f247082a1514 Mon Sep 17 00:00:00 2001 From: Ofer Koren Date: Wed, 2 Feb 2022 17:27:51 +0200 Subject: [PATCH 12/37] Revert "exceptions: ensure ProcessExecutionError can be pickled" This reverts commit e635c16ff61f566b8b9e4ef5105d03a2ca6e04c5. --- plumbum/commands/processes.py | 6 +----- tests/test_local.py | 6 ------ 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index f16b5a33c..0b764cab0 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -114,11 +114,7 @@ class ProcessExecutionError(EnvironmentError): """ def __init__(self, argv, retcode, stdout, stderr, message=None): - - # we can't use 'super' here since EnvironmentError only keeps the first 2 args, - # which leads to failuring in loading this object from a pickle.dumps. - Exception.__init__(self, argv, retcode, stdout, stderr) - + super().__init__(argv, retcode, stdout, stderr) self.message = message self.argv = argv self.retcode = retcode diff --git a/tests/test_local.py b/tests/test_local.py index 448ce0b35..fedcd833e 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -729,12 +729,6 @@ def test_exceptions(self): local.cmd.false() assert repr(exc_info.value) == "ProcessExecutionError(['/usr/bin/false'], 1)" - def test_exception_pickling(self): - import pickle - with pytest.raises(ProcessExecutionError) as exc_info: - local.cmd.ls("no-file") - assert pickle.loads(pickle.dumps(exc_info.value)).argv == exc_info.value.argv - def test_tempdir(self): with local.tempdir() as dir: assert dir.is_dir() From 7bba25587967a5aa0c9f32933bc2a9b59f83bb24 Mon Sep 17 00:00:00 2001 From: Ofer Koren Date: Wed, 2 Feb 2022 17:27:52 +0200 Subject: [PATCH 13/37] Revert "exceptions: fix ProcessExecutionError creating recursion errors" This reverts commit 32e314cb7749c5efaf57739dde8c701b4e5392a6. --- plumbum/commands/processes.py | 2 +- tests/test_local.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 0b764cab0..445d0611e 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -114,7 +114,7 @@ class ProcessExecutionError(EnvironmentError): """ def __init__(self, argv, retcode, stdout, stderr, message=None): - super().__init__(argv, retcode, stdout, stderr) + super().__init__(self, argv, retcode, stdout, stderr) self.message = message self.argv = argv self.retcode = retcode diff --git a/tests/test_local.py b/tests/test_local.py index fedcd833e..27c9439aa 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -724,11 +724,6 @@ def test_quoting(self): ls("-a", "") # check that empty strings are rendered correctly assert execinfo.value.argv[-2:] == ["-a", ""] - def test_exceptions(self): - with pytest.raises(ProcessExecutionError) as exc_info: - local.cmd.false() - assert repr(exc_info.value) == "ProcessExecutionError(['/usr/bin/false'], 1)" - def test_tempdir(self): with local.tempdir() as dir: assert dir.is_dir() From ba4b1a005fe187b46b81ff757f58e16036eb0cd5 Mon Sep 17 00:00:00 2001 From: Ofer Koren Date: Thu, 3 Feb 2022 18:01:22 +0200 Subject: [PATCH 14/37] exceptions: ensure ProcessExecutionError can be pickled (#586) * exceptions: ensure ProcessExecutionError can be pickled * fix: EnvironmentError is OSError in Python 3 * Update plumbum/commands/processes.py Co-authored-by: Henry Schreiner --- plumbum/commands/processes.py | 8 ++++++-- tests/test_local.py | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 445d0611e..c09010177 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -106,7 +106,7 @@ def run(self): # =================================================================================================== # Exceptions # =================================================================================================== -class ProcessExecutionError(EnvironmentError): +class ProcessExecutionError(OSError): """Represents the failure of a process. When the exit code of a terminated process does not match the expected result, this exception is raised by :func:`run_proc `. It contains the process' return code, stdout, and stderr, as @@ -114,7 +114,11 @@ class ProcessExecutionError(EnvironmentError): """ def __init__(self, argv, retcode, stdout, stderr, message=None): - super().__init__(self, argv, retcode, stdout, stderr) + + # we can't use 'super' here since OSError only keeps the first 2 args, + # which leads to failuring in loading this object from a pickle.dumps. + Exception.__init__(self, argv, retcode, stdout, stderr) + self.message = message self.argv = argv self.retcode = retcode diff --git a/tests/test_local.py b/tests/test_local.py index 27c9439aa..f1e528731 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -724,6 +724,13 @@ def test_quoting(self): ls("-a", "") # check that empty strings are rendered correctly assert execinfo.value.argv[-2:] == ["-a", ""] + def test_exception_pickling(self): + import pickle + + with pytest.raises(ProcessExecutionError) as exc_info: + local.cmd.ls("no-file") + assert pickle.loads(pickle.dumps(exc_info.value)).argv == exc_info.value.argv + def test_tempdir(self): with local.tempdir() as dir: assert dir.is_dir() From 4931a15d409e3e5e6a3a89464f946bab457dd141 Mon Sep 17 00:00:00 2001 From: damani42 <56308939+damani42@users.noreply.github.com> Date: Thu, 3 Feb 2022 17:02:18 +0100 Subject: [PATCH 15/37] docs: use f-strings instead format (#585) In the documentation use f-strings in the examples. --- docs/cli.rst | 4 ++-- docs/colorlib.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 3a6b9d4c6..15aee5e98 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -23,7 +23,7 @@ might look like this:: verbose = cli.Flag(["v", "verbose"], help = "If given, I will be very talkative") def main(self, filename): - print("I will now read {0}".format(filename)) + print(f"I will now read {filename}") if self.verbose: print("Yadda " * 200) @@ -492,7 +492,7 @@ attached to the root application using the ``subcommand`` decorator :: def main(self, *args): if args: - print("Unknown command {0!r}".format(args[0])) + print(f"Unknown command {args[0]}") return 1 # error exit code if not self.nested_command: # will be ``None`` if no sub-command follows print("No command given") diff --git a/docs/colorlib.rst b/docs/colorlib.rst index e880b4ba3..46bbb905d 100644 --- a/docs/colorlib.rst +++ b/docs/colorlib.rst @@ -246,9 +246,9 @@ For example, if you wanted to create an HTMLStyle and HTMLcolors, you could do:: result = '' if self.bg and not self.bg.reset: - result += ''.format(self.bg.hex_code) + result += f'' if self.fg and not self.fg.reset: - result += ''.format(self.fg.hex_code) + result += f'' for attr in sorted(self.attributes): if self.attributes[attr]: result += '<' + self.attribute_names[attr] + '>' From 8fdb08d43b312025faf1f626fa8af202a2707fb9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 3 Feb 2022 12:06:36 -0500 Subject: [PATCH 16/37] chore: py2 removed from docs, code (#589) * Includes removals for Python 2.5(!) workarounds. * Always use `python3` over `python` in code and docs. * Using Python 3 print statements in docs. * Avoid showing `u'` in docs. --- README.rst | 4 ++-- docs/_cheatsheet.rst | 30 +++++++++++++++--------------- docs/cli.rst | 18 +++++++++--------- docs/colorlib.rst | 12 ++++++------ docs/colors.rst | 12 ++++++------ docs/local_commands.rst | 10 +++++----- docs/local_machine.rst | 31 ++++++++++++++++--------------- docs/paths.rst | 2 +- docs/remote.rst | 20 ++++++++++---------- examples/alignment.py | 2 +- examples/color.py | 2 +- examples/filecopy.py | 2 +- examples/fullcolor.py | 2 +- examples/geet.py | 12 ++++++------ examples/make_figures.py | 2 +- examples/simple_cli.py | 12 ++++++------ plumbum/__init__.py | 14 +++++++------- plumbum/cli/application.py | 6 ++---- plumbum/cli/image.py | 2 +- plumbum/colorlib/__init__.py | 2 +- plumbum/colorlib/__main__.py | 2 +- plumbum/colorlib/styles.py | 2 +- plumbum/commands/daemons.py | 3 +-- plumbum/fs/atomic.py | 10 ++++------ plumbum/machines/remote.py | 4 ++-- plumbum/path/local.py | 10 +++------- plumbum/path/remote.py | 4 +--- setup.py | 2 +- tests/test_factories.py | 2 +- tests/test_local.py | 16 ++++++++-------- tests/test_remote.py | 8 ++------ tests/test_visual_color.py | 2 +- translations.py | 2 +- 33 files changed, 125 insertions(+), 139 deletions(-) diff --git a/README.rst b/README.rst index 8c1b65721..d06ca5410 100644 --- a/README.rst +++ b/README.rst @@ -84,7 +84,7 @@ Redirection >>> from plumbum.cmd import cat, head >>> ((cat < "setup.py") | head["-n", 4])() - '#!/usr/bin/env python\nimport os\n\ntry:\n' + '#!/usr/bin/env python3\nimport os\n\ntry:\n' >>> (ls["-a"] > "file.list")() '' >>> (cat["file.list"] | wc["-l"])() @@ -174,7 +174,7 @@ Sample output :: - $ python simple_cli.py -v -I foo/bar -Ispam/eggs x.cpp y.cpp z.cpp + $ python3 simple_cli.py -v -I foo/bar -Ispam/eggs x.cpp y.cpp z.cpp Verbose: True Include dirs: ['foo/bar', 'spam/eggs'] Compiling: ('x.cpp', 'y.cpp', 'z.cpp') diff --git a/docs/_cheatsheet.rst b/docs/_cheatsheet.rst index 7334b049f..26a3c8b4c 100644 --- a/docs/_cheatsheet.rst +++ b/docs/_cheatsheet.rst @@ -9,10 +9,10 @@ Basics >>> ls LocalCommand() >>> ls() - u'build.py\ndist\ndocs\nLICENSE\nplumbum\nREADME.rst\nsetup.py\ntests\ntodo.txt\n' + 'build.py\ndist\ndocs\nLICENSE\nplumbum\nREADME.rst\nsetup.py\ntests\ntodo.txt\n' >>> notepad = local["c:\\windows\\notepad.exe"] >>> notepad() # Notepad window pops up - u'' # Notepad window is closed by user, command returns + '' # Notepad window is closed by user, command returns Instead of writing ``xxx = local["xxx"]`` for every program you wish to use, you can also :ref:`import commands `: @@ -28,7 +28,7 @@ Or, use the ``local.cmd`` syntactic-sugar: >>> local.cmd.ls LocalCommand() >>> local.cmd.ls() - u'build.py\ndist\ndocs\nLICENSE\nplumbum\nREADME.rst\nsetup.py\ntests\ntodo.txt\n' + 'build.py\ndist\ndocs\nLICENSE\nplumbum\nREADME.rst\nsetup.py\ntests\ntodo.txt\n' See :ref:`guide-local-commands`. @@ -38,10 +38,10 @@ Piping .. code-block:: python >>> chain = ls["-a"] | grep["-v", "\\.py"] | wc["-l"] - >>> print chain + >>> print(chain) /bin/ls -a | /bin/grep -v '\.py' | /usr/bin/wc -l >>> chain() - u'13\n' + '13\n' See :ref:`guide-local-commands-pipelining`. @@ -51,11 +51,11 @@ Redirection .. code-block:: python >>> ((cat < "setup.py") | head["-n", 4])() - u'#!/usr/bin/env python\nimport os\n\ntry:\n' + '#!/usr/bin/env python3\nimport os\n\ntry:\n' >>> (ls["-a"] > "file.list")() - u'' + '' >>> (cat["file.list"] | wc["-l"])() - u'17\n' + '17\n' See :ref:`guide-local-commands-redir`. @@ -69,7 +69,7 @@ Working-directory manipulation >>> with local.cwd(local.cwd / "docs"): ... chain() ... - u'15\n' + '15\n' A more explicit, and thread-safe way of running a command in a differet directory is using the ``.with_cwd()`` method: @@ -103,7 +103,7 @@ Command nesting .. code-block:: python >>> from plumbum.cmd import sudo - >>> print sudo[ifconfig["-a"]] + >>> print(sudo[ifconfig["-a"]]) /usr/bin/sudo /sbin/ifconfig -a >>> (sudo[ifconfig["-a"]] | grep["-i", "loop"]) & FG lo Link encap:Local Loopback @@ -127,7 +127,7 @@ and `Paramiko `_ (a pure-Python implement >>> with remote.cwd("/lib"): ... (r_ls | grep["0.so.0"])() ... - u'libusb-1.0.so.0\nlibusb-1.0.so.0.0.0\n' + 'libusb-1.0.so.0\nlibusb-1.0.so.0.0.0\n' See :ref:`guide-remote`. @@ -150,9 +150,9 @@ CLI applications logging.root.setLevel(level) def main(self, *srcfiles): - print "Verbose:", self.verbose - print "Include dirs:", self.include_dirs - print "Compiling:", srcfiles + print("Verbose:", self.verbose) + print("Include dirs:", self.include_dirs) + print("Compiling:", srcfiles) if __name__ == "__main__": MyCompiler.run() @@ -162,7 +162,7 @@ Sample output :: - $ python simple_cli.py -v -I foo/bar -Ispam/eggs x.cpp y.cpp z.cpp + $ python3 simple_cli.py -v -I foo/bar -Ispam/eggs x.cpp y.cpp z.cpp Verbose: True Include dirs: ['foo/bar', 'spam/eggs'] Compiling: ('x.cpp', 'y.cpp', 'z.cpp') diff --git a/docs/cli.rst b/docs/cli.rst index 15aee5e98..a85af4309 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -32,10 +32,10 @@ might look like this:: And you can run it:: - $ python example.py foo + $ python3 example.py foo I will now read foo - $ python example.py --help + $ python3 example.py --help example.py v1.0 Usage: example.py [SWITCHES] filename @@ -443,12 +443,12 @@ can be enabled by defining the class-level attribute ``ALLOW_ABBREV`` to True. F With the above definition, running the following will raise an error due to ambiguity:: - $ python example.py --ch # error! matches --cheese and --chives + $ python3 example.py --ch # error! matches --cheese and --chives However, the following two lines are equivalent:: - $ python example.py --che - $ python example.py --cheese + $ python3 example.py --che + $ python3 example.py --cheese .. _guide-subcommands: @@ -525,7 +525,7 @@ attached to the root application using the ``subcommand`` decorator :: Here's an example of running this application:: - $ python geet.py --help + $ python3 geet.py --help geet v1.7.2 The l33t version control @@ -540,7 +540,7 @@ Here's an example of running this application:: push pushes the current local branch to the remote one; see 'geet push --help' for more info - $ python geet.py commit --help + $ python3 geet.py commit --help geet commit v1.7.2 creates a new commit in the current branch @@ -553,7 +553,7 @@ Here's an example of running this application:: -a automatically add changed files -m VALUE:str sets the commit message; required - $ python geet.py commit -m "foo" + $ python3 geet.py commit -m "foo" committing... @@ -605,7 +605,7 @@ is ignored and the image will no longer be constrained to scale proportionately. To directly plot an image, the ``show`` method takes a filename and a double parameter, which doubles the vertical resolution on some fonts. The ``show_pil`` and ``show_pil_double`` methods directly take a PIL-like object. To plot an image -from the command line, the module can be run directly: ``python -m +from the command line, the module can be run directly: ``python3 -m plumbum.cli.image myimage.png``. For the full list of helpers or more information, see the :ref:`api docs `. diff --git a/docs/colorlib.rst b/docs/colorlib.rst index 46bbb905d..123360afb 100644 --- a/docs/colorlib.rst +++ b/docs/colorlib.rst @@ -87,17 +87,17 @@ following, typed into the command line, will restore it: .. code:: bash - $ python -m plumbum.colors + $ python3 -m plumbum.colors This also supports command line access to unsafe color manipulations, such as .. code:: bash - $ python -m plumbum.colors blue - $ python -m plumbum.colors bg red - $ python -m plumbum.colors fg 123 - $ python -m plumbum.colors bg reset - $ python -m plumbum.colors underline + $ python3 -m plumbum.colors blue + $ python3 -m plumbum.colors bg red + $ python3 -m plumbum.colors fg 123 + $ python3 -m plumbum.colors bg reset + $ python3 -m plumbum.colors underline You can use any path or number available as a style. diff --git a/docs/colors.rst b/docs/colors.rst index 36543e94b..cb6c1b2c5 100644 --- a/docs/colors.rst +++ b/docs/colors.rst @@ -164,17 +164,17 @@ when Python exits. .. code:: bash - $ python -m plumbum.colors + $ python3 -m plumbum.colors This also supports command line access to unsafe color manipulations, such as .. code:: bash - $ python -m plumbum.colors blue - $ python -m plumbum.colors bg red - $ python -m plumbum.colors fg 123 - $ python -m plumbum.colors bg reset - $ python -m plumbum.colors underline + $ python3 -m plumbum.colors blue + $ python3 -m plumbum.colors bg red + $ python3 -m plumbum.colors fg 123 + $ python3 -m plumbum.colors bg reset + $ python3 -m plumbum.colors underline You can use any path or number available as a style. diff --git a/docs/local_commands.rst b/docs/local_commands.rst index 9e1d0c724..39b4370e3 100644 --- a/docs/local_commands.rst +++ b/docs/local_commands.rst @@ -93,7 +93,7 @@ Now that we can bind arguments to commands, forming pipelines is easy and straig using ``|`` (bitwise-or):: >>> chain = ls["-l"] | grep[".py"] - >>> print chain + >>> print(chain) C:\Program Files\Git\bin\ls.exe -l | C:\Program Files\Git\bin\grep.exe .py >>> >>> chain() @@ -167,7 +167,7 @@ one you passed:: but not both), you can use the ``run`` method, which will provide all of them >>> cat["non/existing.file"].run(retcode=None) - (1, u'', u'/bin/cat: non/existing.file: No such file or directory\n') + (1, '', '/bin/cat: non/existing.file: No such file or directory\n') @@ -277,11 +277,11 @@ as we've seen above, but they can also be other **commands**! This allows nestin one another, forming complex command objects. The classic example is ``sudo``:: >>> from plumbum.cmd import sudo - >>> print sudo[ls["-l", "-a"]] + >>> print(sudo[ls["-l", "-a"]]) /usr/bin/sudo /bin/ls -l -a >>> sudo[ls["-l", "-a"]]() - u'total 22\ndrwxr-xr-x 8 sebulba Administ 4096 May 9 20:46 .\n[...]' + 'total 22\ndrwxr-xr-x 8 sebulba Administ 4096 May 9 20:46 .\n[...]' In fact, you can nest even command-chains (i.e., pipes and redirections), e.g., ``sudo[ls | grep["\\.py"]]``; however, that would require that the top-level program be able @@ -302,7 +302,7 @@ We'll learn more about remote command execution :ref:`later >> print ssh["somehost", ssh["anotherhost", ls | grep["\\.py"]]] + >>> print(ssh["somehost", ssh["anotherhost", ls | grep["\\.py"]]]) /bin/ssh somehost /bin/ssh anotherhost /bin/ls '|' /bin/grep "'\\.py'" In this example, we first ssh to ``somehost``, from it we ssh to ``anotherhost``, and on that host diff --git a/docs/local_machine.rst b/docs/local_machine.rst index 3b0693c1b..4a282bb14 100644 --- a/docs/local_machine.rst +++ b/docs/local_machine.rst @@ -19,9 +19,9 @@ Another member is ``python``, which is a command object that points to the curre (``sys.executable``):: >>> local.python - - >>> local.python("-c", "import sys;print sys.version") - '2.7.2 (default, Jun 12 2011, 15:08:59) [MSC v.1500 32 bit (Intel)]\r\n' + + >>> local.python("-c", "import sys;print(sys.version)") + '3.10.0 (default, Feb 2 2022, 02:22:22) [MSC v.1931 64 bit (Intel)]\r\n' Working Directory ----------------- @@ -35,14 +35,15 @@ The ``local.cwd`` attribute represents the current working directory. You can ch You can also use it as a *context manager*, so it behaves like ``pushd``/``popd``:: + >>> ls_l = ls | wc["-l"] >>> with local.cwd("c:\\windows"): - ... print "%s:%s" % (local.cwd, (ls | wc["-l"])()) + ... print(f"{local.cwd}:{ls_l()}") ... with local.cwd("c:\\windows\\system32"): - ... print "%s:%s" % (local.cwd, (ls | wc["-l"])()) + ... print(f"{local.cwd}:{ls_l()}") ... c:\windows: 105 c:\windows\system32: 3013 - >>> print "%s:%s" % (local.cwd, (ls | wc["-l"])()) + >>> print(f"{local.cwd}:{ls_l()}") d:\workspace\plumbum: 9 Finally, A more explicit and thread-safe way of running a command in a different directory is using the ``.with_cwd()`` method: @@ -65,10 +66,10 @@ And similarity to ``cwd`` is the context-manager nature of ``env``; each level w it's own private copy of the environment:: >>> with local.env(FOO="BAR"): - ... local.python("-c", "import os;print os.environ['FOO']") + ... local.python("-c", "import os; print(os.environ['FOO'])") ... with local.env(FOO="SPAM"): - ... local.python("-c", "import os;print os.environ['FOO']") - ... local.python("-c", "import os;print os.environ['FOO']") + ... local.python("-c", "import os; print(os.environ['FOO'])") + ... local.python("-c", "import os; print(os.environ['FOO'])") ... 'BAR\r\n' 'SPAM\r\n' @@ -77,10 +78,10 @@ it's own private copy of the environment:: Traceback (most recent call last): [...] ProcessExecutionError: Unexpected exit code: 1 - Command line: | /usr/bin/python -c "import os;print(os.environ['FOO'])" + Command line: | /usr/bin/python3 -c "import os; print(os.environ['FOO'])" Stderr: | Traceback (most recent call last): | File "", line 1, in - | File "/usr/lib/python3.5/os.py", line 725, in __getitem__ + | File "/usr/lib/python3.10/os.py", line 725, in __getitem__ | raise KeyError(key) from None | KeyError: 'FOO' @@ -93,13 +94,13 @@ properties for getting the username (``.user``), the home path (``.home``), and >>> local.env.home >>> local.env.path - [, , ...] + [, , ...] >>> >>> local.which("python") - - >>> local.env.path.insert(0, "c:\\python32") + + >>> local.env.path.insert(0, "c:\\python310") >>> local.which("python") - + For further information, see the :ref:`api docs `. diff --git a/docs/paths.rst b/docs/paths.rst index 12c3724f4..d8e0177ce 100644 --- a/docs/paths.rst +++ b/docs/paths.rst @@ -60,7 +60,7 @@ Paths can be composed using ``/`` or ``[]``:: You can also iterate over directories to get the contents:: >>> for p2 in p: - ... print p2 + ... print(p2) ... c:\windows\addins c:\windows\appcompat diff --git a/docs/remote.rst b/docs/remote.rst index 9b4dbae2f..7f3c5b7e9 100644 --- a/docs/remote.rst +++ b/docs/remote.rst @@ -55,7 +55,7 @@ counterparts, they can be used as context managers, so their effects can be cont >>> rem.cwd >>> with rem.cwd(rem.cwd / "Desktop"): - ... print rem.cwd + ... print(rem.cwd) /home/john/Desktop >>> rem.env["PATH"] /bin:/sbin:/usr/bin:/usr/local/bin @@ -102,7 +102,7 @@ object. You can either pass the command's name, in which case it will be resolve >>> r_ls = rem["ls"] >>> r_grep = rem["grep"] >>> r_ls() - u'foo\nbar\spam\n' + 'foo\nbar\spam\n' Nesting Commands ^^^^^^^^^^^^^^^^ @@ -111,7 +111,7 @@ behind the scenes - it nests each command inside ``ssh``. Here are some examples >>> r_sudo = rem["sudo"] >>> r_ifconfig = rem["ifconfig"] - >>> print r_sudo[r_ifconfig["-a"]]() + >>> print(r_sudo[r_ifconfig["-a"]]()) eth0 Link encap:Ethernet HWaddr ... [...] @@ -119,11 +119,11 @@ You can nest multiple commands, one within another. For instance, you can connec over SSH and use that machine's SSH client to connect to yet another machine. Here's a sketch:: >>> from plumbum.cmd import ssh - >>> print ssh["localhost", ssh["localhost", "ls"]] + >>> print(ssh["localhost", ssh["localhost", "ls"]]) /usr/bin/ssh localhost /usr/bin/ssh localhost ls >>> >>> ssh["localhost", ssh["localhost", "ls"]]() - u'bin\nDesktop\nDocuments\n...' + 'bin\nDesktop\nDocuments\n...' Piping @@ -134,7 +134,7 @@ place on the local machine! Consider this code for instance :: >>> r_grep = rem["grep"] >>> r_ls = rem["ls"] >>> (r_ls | r_grep["b"])() - u'bin\nPublic\n' + 'bin\nPublic\n' Although ``r_ls`` and ``r_grep`` are remote commands, the data is sent from ``r_ls`` to the local machine, which then sends it to the remote one for running ``grep``. This will be fixed in a future @@ -145,7 +145,7 @@ For example, the previous code can be written as :: >>> from plumbum.cmd import grep >>> (r_ls | grep["b"])() - u'bin\nPublic\n' + 'bin\nPublic\n' Which is even more efficient (no need to send data back and forth over SSH). @@ -178,9 +178,9 @@ and it works along the lines of the ``SshMachine``:: RemoteCommand(, ) >>> r_ls = rem["ls"] >>> r_ls() - u'bin\nDesktop\nDocuments\nDownloads\nexamples.desktop\nMusic\nPictures\n...' + 'bin\nDesktop\nDocuments\nDownloads\nexamples.desktop\nMusic\nPictures\n...' >>> r_ls("-a") - u'.\n..\n.adobe\n.bash_history\n.bash_logout\n.bashrc\nbin...' + '.\n..\n.adobe\n.bash_history\n.bash_logout\n.bashrc\nbin...' .. note:: Using ``ParamikoMachine`` requires paramiko to be installed on your system. Also, you have @@ -209,7 +209,7 @@ object >>> s = mach.session() >>> s.run("ls | grep b") - (0, u'bin\nPublic\n', u'') + (0, 'bin\nPublic\n', '') Tunneling Example diff --git a/examples/alignment.py b/examples/alignment.py index f5729b726..2803a0290 100755 --- a/examples/alignment.py +++ b/examples/alignment.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from plumbum import cli diff --git a/examples/color.py b/examples/color.py index 520b3a7ca..649d27028 100755 --- a/examples/color.py +++ b/examples/color.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from plumbum import colors diff --git a/examples/filecopy.py b/examples/filecopy.py index 7e5940b6e..0ab66357d 100755 --- a/examples/filecopy.py +++ b/examples/filecopy.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import logging from plumbum import cli, local diff --git a/examples/fullcolor.py b/examples/fullcolor.py index 2e12e74a6..aac32597d 100755 --- a/examples/fullcolor.py +++ b/examples/fullcolor.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from plumbum import colors diff --git a/examples/geet.py b/examples/geet.py index 484c89041..f09fa1402 100755 --- a/examples/geet.py +++ b/examples/geet.py @@ -1,14 +1,14 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Examples:: - $ python geet.py + $ python3 geet.py no command given - $ python geet.py leet + $ python3 geet.py leet unknown command 'leet' - $ python geet.py --help + $ python3 geet.py --help geet v1.7.2 The l33t version control @@ -23,7 +23,7 @@ push pushes the current local branch to the remote one; see 'geet push --help' for more info - $ python geet.py commit --help + $ python3 geet.py commit --help geet commit v1.7.2 creates a new commit in the current branch @@ -36,7 +36,7 @@ -a automatically add changed files -m VALUE:str sets the commit message; required - $ python geet.py commit -m "foo" + $ python3 geet.py commit -m "foo" committing... """ diff --git a/examples/make_figures.py b/examples/make_figures.py index 16dd2310c..f13d80c3d 100755 --- a/examples/make_figures.py +++ b/examples/make_figures.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from plumbum import FG, cli, local from plumbum.cmd import convert, pdflatex diff --git a/examples/simple_cli.py b/examples/simple_cli.py index 8fa3635f0..24095b226 100755 --- a/examples/simple_cli.py +++ b/examples/simple_cli.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ -$ python simple_cli.py --help +$ python3 simple_cli.py --help simple_cli.py v1.0 Usage: simple_cli.py [SWITCHES] srcfiles... @@ -14,22 +14,22 @@ --loglevel LEVEL:int Sets the log-level of the logger -v, --verbose Enable verbose mode -$ python simple_cli.py x.cpp y.cpp z.cpp +$ python3 simple_cli.py x.cpp y.cpp z.cpp Verbose: False Include dirs: [] Compiling: ('x.cpp', 'y.cpp', 'z.cpp') -$ python simple_cli.py -v +$ python3 simple_cli.py -v Verbose: True Include dirs: [] Compiling: () -$ python simple_cli.py -v -Ifoo/bar -Ispam/eggs +$ python3 simple_cli.py -v -Ifoo/bar -Ispam/eggs Verbose: True Include dirs: ['foo/bar', 'spam/eggs'] Compiling: () -$ python simple_cli.py -v -I foo/bar -Ispam/eggs x.cpp y.cpp z.cpp +$ python3 simple_cli.py -v -I foo/bar -Ispam/eggs x.cpp y.cpp z.cpp Verbose: True Include dirs: ['foo/bar', 'spam/eggs'] Compiling: ('x.cpp', 'y.cpp', 'z.cpp') diff --git a/plumbum/__init__.py b/plumbum/__init__.py index 47f53c699..f7bda3f8d 100644 --- a/plumbum/__init__.py +++ b/plumbum/__init__.py @@ -6,16 +6,16 @@ >>> from plumbum.cmd import ls, grep, wc, cat >>> ls() - u'build.py\ndist\ndocs\nLICENSE\nplumbum\nREADME.rst\nsetup.py\ntests\ntodo.txt\n' + 'build.py\ndist\ndocs\nLICENSE\nplumbum\nREADME.rst\nsetup.py\ntests\ntodo.txt\n' >>> chain = ls["-a"] | grep["-v", "py"] | wc["-l"] - >>> print chain + >>> print(chain) /bin/ls -a | /bin/grep -v py | /usr/bin/wc -l >>> chain() - u'12\n' + '12\n' >>> ((ls["-a"] | grep["-v", "py"]) > "/tmp/foo.txt")() - u'' + '' >>> ((cat < "/tmp/foo.txt") | wc["-l"])() - u'12\n' + '12\n' >>> from plumbum import local, FG, BG >>> with local.cwd("/tmp"): ... (ls | wc["-l"]) & FG @@ -23,9 +23,9 @@ 13 # printed directly to the interpreter's stdout >>> (ls | wc["-l"]) & BG - >>> f=_ + >>> f = _ >>> f.stdout # will wait for the process to terminate - u'9\n' + '9\n' Plumbum includes local/remote path abstraction, working directory and environment manipulation, process execution, remote process execution over SSH, tunneling, diff --git a/plumbum/cli/application.py b/plumbum/cli/application.py index 47d6059a4..f5e98393a 100644 --- a/plumbum/cli/application.py +++ b/plumbum/cli/application.py @@ -445,8 +445,7 @@ def _handle_argument(val, argtype, name): if argtype: try: return argtype(val) - except (TypeError, ValueError): - ex = sys.exc_info()[1] # compat + except (TypeError, ValueError) as ex: raise WrongArgumentType( T_( "Argument of {name} expected to be {argtype}, not {val!r}:\n {ex!r}" @@ -608,8 +607,7 @@ def run( inst.helpall() except ShowVersion: inst.version() - except SwitchError: - ex = sys.exc_info()[1] # compatibility with python 2.5 + except SwitchError as ex: print(T_("Error: {0}").format(ex)) print(T_("------")) inst.help() diff --git a/plumbum/cli/image.py b/plumbum/cli/image.py index ee4ed792c..293b606d2 100644 --- a/plumbum/cli/image.py +++ b/plumbum/cli/image.py @@ -56,7 +56,7 @@ def show_pil(self, im): for y in range(size[1]): for x in range(size[0] - 1): pix = new_im.getpixel((x, y)) - print(colors.bg.rgb(*pix), " ", sep="", end="") # u'\u2588' + print(colors.bg.rgb(*pix), " ", sep="", end="") # '\u2588' print(colors.reset, " ", sep="") print(colors.reset) diff --git a/plumbum/colorlib/__init__.py b/plumbum/colorlib/__init__.py index 846e5e0b2..e80bd10b7 100644 --- a/plumbum/colorlib/__init__.py +++ b/plumbum/colorlib/__init__.py @@ -38,7 +38,7 @@ def load_ipython_extension(ipython): # pragma: no cover def main(): # pragma: no cover """Color changing script entry. Call using - python -m plumbum.colors, will reset if no arguments given.""" + python3 -m plumbum.colors, will reset if no arguments given.""" color = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else "" ansicolors.use_color = True ansicolors.get_colors_from_string(color).now() diff --git a/plumbum/colorlib/__main__.py b/plumbum/colorlib/__main__.py index b06ab2fb3..47d47b123 100644 --- a/plumbum/colorlib/__main__.py +++ b/plumbum/colorlib/__main__.py @@ -1,6 +1,6 @@ """ This is provided as a quick way to recover your terminal. Simply run -``python -m plumbum.colorlib`` +``python3 -m plumbum.colorlib`` to recover terminal color. """ diff --git a/plumbum/colorlib/styles.py b/plumbum/colorlib/styles.py index eed9838e5..afb842dfc 100644 --- a/plumbum/colorlib/styles.py +++ b/plumbum/colorlib/styles.py @@ -510,7 +510,7 @@ def print(self, *printables, **kargs): file.flush() print_ = print - """Shortcut just in case user not using __future__""" + """DEPRECATED: Shortcut from classic Python 2""" def __getitem__(self, wrapped): """The [] syntax is supported for wrapping""" diff --git a/plumbum/commands/daemons.py b/plumbum/commands/daemons.py index 4c1de6bb6..d41ca2fdc 100644 --- a/plumbum/commands/daemons.py +++ b/plumbum/commands/daemons.py @@ -90,8 +90,7 @@ def poll(self=proc): if self.returncode is None: try: os.kill(self.pid, 0) - except OSError: - ex = sys.exc_info()[1] + except OSError as ex: if ex.errno == errno.ESRCH: # process does not exist self.returncode = 0 diff --git a/plumbum/fs/atomic.py b/plumbum/fs/atomic.py index 16954c716..5aebc80fa 100644 --- a/plumbum/fs/atomic.py +++ b/plumbum/fs/atomic.py @@ -4,7 +4,6 @@ import atexit import os -import sys import threading from contextlib import contextmanager @@ -35,8 +34,7 @@ def locked_file(fileno, blocking=True): 0xFFFFFFFF, OVERLAPPED(), ) - except WinError: - _, ex, _ = sys.exc_info() + except WinError as ex: raise OSError(*ex.args) from None try: yield @@ -185,9 +183,9 @@ class AtomicCounterFile: Example:: acf = AtomicCounterFile.open("/some/file") - print acf.next() # e.g., 7 - print acf.next() # 8 - print acf.next() # 9 + print(acf.next()) # e.g., 7 + print(acf.next()) # 8 + print(acf.next()) # 9 .. versionadded:: 1.3 """ diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index d379765ac..df86fa42a 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -180,7 +180,7 @@ def _get_uname(self): return out.strip() rc, out, _ = self._session.run( - "python -c 'import platform;print(platform.uname()[0])'", retcode=None + "python3 -c 'import platform;print(platform.uname()[0])'", retcode=None ) if rc == 0: return out.strip() @@ -264,7 +264,7 @@ def __getitem__(self, cmd): def python(self): """A command that represents the default remote python interpreter""" if not self._python: - self._python = self["python"] + self._python = self["python3"] return self._python def session(self, isatty=False, new_session=False): diff --git a/plumbum/path/local.py b/plumbum/path/local.py index 38edf5ba0..bdaab3c06 100644 --- a/plumbum/path/local.py +++ b/plumbum/path/local.py @@ -3,7 +3,6 @@ import logging import os import shutil -import sys import urllib.parse as urlparse import urllib.request as urllib from contextlib import contextmanager @@ -161,9 +160,8 @@ def delete(self): else: try: os.remove(str(self)) - except OSError: # pragma: no cover + except OSError as ex: # pragma: no cover # file might already been removed (a race with other threads/processes) - _, ex, _ = sys.exc_info() if ex.errno != errno.ENOENT: raise @@ -197,9 +195,8 @@ def mkdir(self, mode=0o777, parents=True, exist_ok=True): os.makedirs(str(self), mode) else: os.mkdir(str(self), mode) - except OSError: # pragma: no cover + except OSError as ex: # pragma: no cover # directory might already exist (a race with other threads/processes) - _, ex, _ = sys.exc_info() if ex.errno != errno.EEXIST or not exist_ok: raise @@ -299,9 +296,8 @@ def unlink(self): else: # windows: use rmdir for directories and directory symlinks os.rmdir(str(self)) - except OSError: # pragma: no cover + except OSError as ex: # pragma: no cover # file might already been removed (a race with other threads/processes) - _, ex, _ = sys.exc_info() if ex.errno != errno.ENOENT: raise diff --git a/plumbum/path/remote.py b/plumbum/path/remote.py index b957b64ba..0741e2cf9 100644 --- a/plumbum/path/remote.py +++ b/plumbum/path/remote.py @@ -1,6 +1,5 @@ import errno import os -import sys import urllib.request as urllib from contextlib import contextmanager @@ -224,8 +223,7 @@ def mkdir(self, mode=None, parents=True, exist_ok=True): self.remote._path_mkdir(self.parent, mode=mode, minus_p=True) try: self.remote._path_mkdir(self, mode=mode, minus_p=False) - except ProcessExecutionError: - _, ex, _ = sys.exc_info() + except ProcessExecutionError as ex: if "File exists" not in ex.stderr: raise diff --git a/setup.py b/setup.py index beda28e82..229b2ebbb 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from setuptools import setup diff --git a/tests/test_factories.py b/tests/test_factories.py index 4b976960e..f0aff1c12 100644 --- a/tests/test_factories.py +++ b/tests/test_factories.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import pytest diff --git a/tests/test_local.py b/tests/test_local.py index f1e528731..dc76a7bc3 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -428,22 +428,22 @@ def test_env(self): local.python("-c", "import os;os.environ['FOOBAR72']") local.env["FOOBAR72"] = "spAm" assert local.python( - "-c", "import os;print (os.environ['FOOBAR72'])" + "-c", "import os; print(os.environ['FOOBAR72'])" ).splitlines() == ["spAm"] with local.env(FOOBAR73=1889): assert local.python( - "-c", "import os;print (os.environ['FOOBAR73'])" + "-c", "import os; print(os.environ['FOOBAR73'])" ).splitlines() == ["1889"] with local.env(FOOBAR73=1778): assert local.python( - "-c", "import os;print (os.environ['FOOBAR73'])" + "-c", "import os; print(os.environ['FOOBAR73'])" ).splitlines() == ["1778"] assert local.python( - "-c", "import os;print (os.environ['FOOBAR73'])" + "-c", "import os; print(os.environ['FOOBAR73'])" ).splitlines() == ["1889"] with pytest.raises(ProcessExecutionError): - local.python("-c", "import os;os.environ['FOOBAR73']") + local.python("-c", "import os; os.environ['FOOBAR73']") # path manipulation with pytest.raises(CommandNotFound): @@ -459,7 +459,7 @@ def test_local(self): assert local.path("foo") == os.path.join(os.getcwd(), "foo") local.which("ls") local["ls"] - assert local.python("-c", "print ('hi there')").splitlines() == ["hi there"] + assert local.python("-c", "print('hi there')").splitlines() == ["hi there"] @skip_on_windows def test_piping(self): @@ -846,7 +846,7 @@ def test_atomic_file(self): def test_atomic_file2(self): af = AtomicFile("tmp.txt") - code = """from __future__ import with_statement + code = """\ from plumbum.fs.atomic import AtomicFile af = AtomicFile("tmp.txt") try: @@ -863,7 +863,7 @@ def test_atomic_file2(self): @skip_on_windows def test_pid_file(self): - code = """from __future__ import with_statement + code = """\ from plumbum.fs.atomic import PidFile, PidFileTaken try: with PidFile("mypid"): diff --git a/tests/test_remote.py b/tests/test_remote.py index 7e94aa53c..19245daa5 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,7 +1,6 @@ import logging import os import socket -import sys import time from multiprocessing import Queue from threading import Thread @@ -421,11 +420,8 @@ def test_iter_lines_timeout(self): ): print("out:", out) print("err:", err) - except NotImplementedError: - try: - pytest.skip(str(sys.exc_info()[1])) - except AttributeError: - return + except NotImplementedError as err: + pytest.skip(str(err)) except ProcessTimedOut: assert i > 3 else: diff --git a/tests/test_visual_color.py b/tests/test_visual_color.py index 5f0cecb25..6ca244289 100644 --- a/tests/test_visual_color.py +++ b/tests/test_visual_color.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import unittest diff --git a/translations.py b/translations.py index 157b5249b..4b984de7c 100755 --- a/translations.py +++ b/translations.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # If you are on macOS and using brew, you might need the following first: # export PATH="/usr/local/opt/gettext/bin:$PATH" From b775d363f7cbff22f6795e2f8606a8f68d7d2693 Mon Sep 17 00:00:00 2001 From: Ofer Koren Date: Thu, 3 Feb 2022 19:07:46 +0200 Subject: [PATCH 17/37] feat: cache commands and remember host in exceptions (#583) * tests: shorten test_timeout to 3s instead of 10s * command: fix handling of env-vars passed to plumbum Commands; support new with_cwd - don't use the non-threadsafe and session-dependent .env() context manager - sync popen support for 'env' param in all machine impls: local/ssh/paramiko - add .with_cwd() to complement .with_env() * ssh: better error reporting on SshSession errors * machines: use a cache to speed-up lookups commands/programs this is particularly important for remote machines, where this lookup is very costly * make iter_lines deal with decoding errors during iteration; Also... remove broken support for no-decoding * Add 'host' to ssh exceptions * .gitignore: add .eggs * iter_lines: added new 'buffer_size' parameter, and updated docstrings * iter_lines: pylint formatting fix * Update plumbum/commands/processes.py * Update base.py Co-authored-by: ErezH Co-authored-by: Henry Schreiner Co-authored-by: Henry Schreiner --- .gitignore | 1 + plumbum/commands/processes.py | 5 ++++- plumbum/machines/base.py | 9 +++++++++ plumbum/machines/local.py | 11 +++++++++++ plumbum/machines/remote.py | 9 +++++++++ plumbum/machines/session.py | 14 ++++++++++++-- plumbum/machines/ssh_machine.py | 2 ++ 7 files changed, 48 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3905ee6bc..6870fe28b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,6 @@ tests/.cache/* *.pot /*venv* *.mypy_cache +.eggs /plumbum/version.py /tests/nohup.out diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index c09010177..da33b76e5 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -113,13 +113,14 @@ class ProcessExecutionError(OSError): well as the command line used to create the process (``argv``) """ - def __init__(self, argv, retcode, stdout, stderr, message=None): + def __init__(self, argv, retcode, stdout, stderr, message=None, *, host=None): # we can't use 'super' here since OSError only keeps the first 2 args, # which leads to failuring in loading this object from a pickle.dumps. Exception.__init__(self, argv, retcode, stdout, stderr) self.message = message + self.host = host self.argv = argv self.retcode = retcode if isinstance(stdout, bytes): @@ -143,6 +144,8 @@ def __str__(self): lines = ["Unexpected exit code: ", str(self.retcode)] cmd = "\n | ".join(cmd.splitlines()) lines += ["\nCommand line: | ", cmd] + if self.host: + lines += ["\nHost: | ", self.host] if stdout: lines += ["\nStdout: | ", stdout] if stderr: diff --git a/plumbum/machines/base.py b/plumbum/machines/base.py index a498c7c2f..754852bee 100644 --- a/plumbum/machines/base.py +++ b/plumbum/machines/base.py @@ -91,3 +91,12 @@ def __getattr__(self, name): @property def cmd(self): return self.Cmd(self) + + def clear_program_cache(self): + """ + Clear the program cache, which is populated via ``machine.which(progname)`` calls. + + This cache speeds up the lookup of a program in the machines PATH, and is particularly + effective for RemoteMachines. + """ + self._program_cache.clear() diff --git a/plumbum/machines/local.py b/plumbum/machines/local.py index 723c72c53..a9a4e5f41 100644 --- a/plumbum/machines/local.py +++ b/plumbum/machines/local.py @@ -8,6 +8,7 @@ from contextlib import contextmanager from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import Dict, Tuple from plumbum.commands import CommandNotFound, ConcreteCommand from plumbum.commands.daemons import posix_daemonize, win32_daemonize @@ -141,6 +142,7 @@ class LocalMachine(BaseMachine): custom_encoding = sys.getfilesystemencoding() uname = platform.uname()[0] + _program_cache: Dict[Tuple[str, str], LocalPath] = {} def __init__(self): self._as_user_stack = [] @@ -182,6 +184,14 @@ def which(cls, progname): :returns: A :class:`LocalPath ` """ + + key = (progname, cls.env.get("PATH", "")) + + try: + return cls._program_cache[key] + except KeyError: + pass + alternatives = [progname] if "_" in progname: alternatives.append(progname.replace("_", "-")) @@ -189,6 +199,7 @@ def which(cls, progname): for pn in alternatives: path = cls._which(pn) if path: + cls._program_cache[key] = path return path raise CommandNotFound(progname, list(cls.env.path)) diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index df86fa42a..097652d51 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -173,6 +173,7 @@ def __init__(self, encoding="utf8", connect_timeout=10, new_session=False): self.uname = self._get_uname() self.env = RemoteEnv(self) self._python = None + self._program_cache = {} def _get_uname(self): rc, out, _ = self._session.run("uname", retcode=None) @@ -225,6 +226,13 @@ def which(self, progname): :returns: A :class:`RemotePath ` """ + key = (progname, self.env.get("PATH", "")) + + try: + return self._program_cache[key] + except KeyError: + pass + alternatives = [progname] if "_" in progname: alternatives.append(progname.replace("_", "-")) @@ -233,6 +241,7 @@ def which(self, progname): for p in self.env.path: fn = p / name if fn.access("x") and not fn.is_dir(): + self._program_cache[key] = fn return fn raise CommandNotFound(progname, self.env.path) diff --git a/plumbum/machines/session.py b/plumbum/machines/session.py index 21d132176..f292ededc 100644 --- a/plumbum/machines/session.py +++ b/plumbum/machines/session.py @@ -72,7 +72,8 @@ class SessionPopen(PopenAddons): """A shell-session-based ``Popen``-like object (has the following attributes: ``stdin``, ``stdout``, ``stderr``, ``returncode``)""" - def __init__(self, proc, argv, isatty, stdin, stdout, stderr, encoding): + def __init__(self, proc, argv, isatty, stdin, stdout, stderr, encoding, *, host): + self.host = host self.proc = proc self.argv = argv self.isatty = isatty @@ -132,6 +133,7 @@ def communicate(self, input=None): # pylint: disable=redefined-builtin stdout, stderr, message="Incorrect username or password provided", + host=self.host, ) from None if returncode == 6: raise HostPublicKeyUnknown( @@ -140,6 +142,7 @@ def communicate(self, input=None): # pylint: disable=redefined-builtin stdout, stderr, message="The authenticity of the host can't be established", + host=self.host, ) from None if returncode != 0: raise SSHCommsError( @@ -148,6 +151,7 @@ def communicate(self, input=None): # pylint: disable=redefined-builtin stdout, stderr, message="SSH communication failed", + host=self.host, ) from None if name == "2": raise SSHCommsChannel2Error( @@ -156,6 +160,7 @@ def communicate(self, input=None): # pylint: disable=redefined-builtin stdout, stderr, message="No stderr result detected. Does the remote have Bash as the default shell?", + host=self.host, ) from None raise SSHCommsError( @@ -164,6 +169,7 @@ def communicate(self, input=None): # pylint: disable=redefined-builtin stdout, stderr, message="No communication channel detected. Does the remote exist?", + host=self.host, ) from err if not line: del sources[i] @@ -202,7 +208,10 @@ class ShellSession: is seen, the shell process is killed """ - def __init__(self, proc, encoding="auto", isatty=False, connect_timeout=5): + def __init__( + self, proc, encoding="auto", isatty=False, connect_timeout=5, *, host=None + ): + self.host = host self.proc = proc self.custom_encoding = proc.custom_encoding if encoding == "auto" else encoding self.isatty = isatty @@ -299,6 +308,7 @@ def popen(self, cmd): MarkedPipe(self.proc.stdout, marker), MarkedPipe(self.proc.stderr, marker), self.custom_encoding, + host=self.host, ) return self._current diff --git a/plumbum/machines/ssh_machine.py b/plumbum/machines/ssh_machine.py index 8ce41c437..648b0efbf 100644 --- a/plumbum/machines/ssh_machine.py +++ b/plumbum/machines/ssh_machine.py @@ -104,6 +104,7 @@ def __init__( scp_args = [] ssh_args = [] + self.host = host if user: self._fqhost = f"{user}@{host}" else: @@ -208,6 +209,7 @@ def session(self, isatty=False, new_session=False): self.custom_encoding, isatty, self.connect_timeout, + host=self.host, ) def tunnel( From cdd0e4ca7d0427f7e0bcc34d31f291bd8245b345 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Langlois Date: Thu, 3 Feb 2022 12:08:18 -0500 Subject: [PATCH 18/37] fix: call to repr() in the French translation was broken (#588) --- plumbum/cli/i18n/fr.po | 2 +- .../cli/i18n/fr/LC_MESSAGES/plumbum.cli.mo | Bin 3614 -> 3614 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/plumbum/cli/i18n/fr.po b/plumbum/cli/i18n/fr.po index 44c2b71e4..af76a4ce6 100644 --- a/plumbum/cli/i18n/fr.po +++ b/plumbum/cli/i18n/fr.po @@ -76,7 +76,7 @@ msgid "" "Argument of {name} expected to be {argtype}, not {val!r}:\n" " {ex!r}" msgstr "" -"Argument de {name} doit être {argtype} , et non {val|1}:\n" +"Argument de {name} doit être {argtype} , et non {val!r}:\n" " {ex!r}" #: plumbum/cli/application.py:461 diff --git a/plumbum/cli/i18n/fr/LC_MESSAGES/plumbum.cli.mo b/plumbum/cli/i18n/fr/LC_MESSAGES/plumbum.cli.mo index 4f38a4e434871e25b6b480577775ae023d4075af..a21ea27c2921c0fce2b192c57eb0119aafa69e37 100644 GIT binary patch delta 15 WcmbOyGf!rNDHoGs(Pne5=ga^o7zEb< delta 15 WcmbOyGf!rNDHl_X;bwEL=ga^p4+QZ5 From 3bca2d4a2dbd38c75c9a8d8d719249f84b0c7cda Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 3 Feb 2022 12:50:32 -0500 Subject: [PATCH 19/37] ci: nicer pylint annotations (#584) * ci: nicer pylint output * style: use math.inf * chore: bump versions, including non-beta Black * fix: hide or fix all pylint warnings --- .github/matchers/pylint.json | 32 ++++++++++ .github/workflows/ci.yml | 4 +- .pre-commit-config.yaml | 8 +-- plumbum/commands/base.py | 7 +++ plumbum/commands/daemons.py | 93 ++++++++++++++-------------- plumbum/commands/processes.py | 7 ++- plumbum/machines/local.py | 5 +- plumbum/machines/paramiko_machine.py | 3 + plumbum/machines/remote.py | 2 +- pyproject.toml | 4 +- 10 files changed, 107 insertions(+), 58 deletions(-) create mode 100644 .github/matchers/pylint.json diff --git a/.github/matchers/pylint.json b/.github/matchers/pylint.json new file mode 100644 index 000000000..e3a6bd16b --- /dev/null +++ b/.github/matchers/pylint.json @@ -0,0 +1,32 @@ +{ + "problemMatcher": [ + { + "severity": "warning", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): ([A-DF-Z]\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-warning" + }, + { + "severity": "error", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): (E\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-error" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1faba0c8..1fb2d7599 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,9 @@ jobs: - uses: actions/setup-python@v2 - uses: pre-commit/action@v2.0.3 - name: pylint - run: pipx run nox -s pylint + run: | + echo "::add-matcher::$GITHUB_WORKSPACE/.github/matchers/pylint.json" + pipx run nox -s pylint tests: name: Tests on 🐍 ${{ matrix.python-version }} ${{ matrix.os }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3648599c..a95da7295 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.1.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -20,7 +20,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/psf/black - rev: "21.12b0" + rev: "22.1.0" hooks: - id: black @@ -30,7 +30,7 @@ repos: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: "v2.29.1" + rev: "v2.31.0" hooks: - id: pyupgrade args: ["--py36-plus"] @@ -55,7 +55,7 @@ repos: additional_dependencies: [flake8-bugbear, flake8-print, flake8-2020] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.930" + rev: "v0.931" hooks: - id: mypy files: plumbum diff --git a/plumbum/commands/base.py b/plumbum/commands/base.py index c15c3ba7f..5650ce3fd 100644 --- a/plumbum/commands/base.py +++ b/plumbum/commands/base.py @@ -602,3 +602,10 @@ def formulate(self, level=0, args=()): # if self.custom_encoding: # argv = [a.encode(self.custom_encoding) for a in argv if isinstance(a, str)] return argv + + @property + def machine(self): + raise NotImplementedError() + + def popen(self, args=(), **kwargs): + raise NotImplementedError() diff --git a/plumbum/commands/daemons.py b/plumbum/commands/daemons.py index d41ca2fdc..82047dd6a 100644 --- a/plumbum/commands/daemons.py +++ b/plumbum/commands/daemons.py @@ -21,7 +21,6 @@ def release(): pass -# pylint: disable-next: inconsistent-return-statements def posix_daemonize(command, cwd, stdout=None, stderr=None, append=True): if stdout is None: stdout = os.devnull @@ -33,7 +32,7 @@ def posix_daemonize(command, cwd, stdout=None, stderr=None, append=True): argv = command.formulate() firstpid = os.fork() if firstpid == 0: - # first child: become session leader, + # first child: become session leader os.close(rfd) rc = 0 try: @@ -58,55 +57,55 @@ def posix_daemonize(command, cwd, stdout=None, stderr=None, append=True): finally: os.close(wfd) os._exit(rc) + + # wait for first child to die + os.close(wfd) + _, rc = os.waitpid(firstpid, 0) + output = os.read(rfd, MAX_SIZE) + os.close(rfd) + try: + output = output.decode("utf8") + except UnicodeError: + pass + if rc == 0 and output.isdigit(): + secondpid = int(output) else: - # wait for first child to die - os.close(wfd) - _, rc = os.waitpid(firstpid, 0) - output = os.read(rfd, MAX_SIZE) - os.close(rfd) - try: - output = output.decode("utf8") - except UnicodeError: - pass - if rc == 0 and output.isdigit(): - secondpid = int(output) - else: - raise ProcessExecutionError(argv, rc, "", output) - proc = subprocess.Popen.__new__(subprocess.Popen) - proc._child_created = True - proc.returncode = None - proc.stdout = None - proc.stdin = None - proc.stderr = None - proc.pid = secondpid - proc.universal_newlines = False - proc._input = None - proc._waitpid_lock = _fake_lock() - proc._communication_started = False - proc.args = argv - proc.argv = argv + raise ProcessExecutionError(argv, rc, "", output) + proc = subprocess.Popen.__new__(subprocess.Popen) + proc._child_created = True + proc.returncode = None + proc.stdout = None + proc.stdin = None + proc.stderr = None + proc.pid = secondpid + proc.universal_newlines = False + proc._input = None + proc._waitpid_lock = _fake_lock() + proc._communication_started = False + proc.args = argv + proc.argv = argv - def poll(self=proc): - if self.returncode is None: - try: - os.kill(self.pid, 0) - except OSError as ex: - if ex.errno == errno.ESRCH: - # process does not exist - self.returncode = 0 - else: - raise - return self.returncode + def poll(self=proc): + if self.returncode is None: + try: + os.kill(self.pid, 0) + except OSError as ex: + if ex.errno == errno.ESRCH: + # process does not exist + self.returncode = 0 + else: + raise + return self.returncode - def wait(self=proc): - while self.returncode is None: - if self.poll() is None: - time.sleep(0.5) - return proc.returncode + def wait(self=proc): + while self.returncode is None: + if self.poll() is None: + time.sleep(0.5) + return proc.returncode - proc.poll = poll - proc.wait = wait - return proc + proc.poll = poll + proc.wait = wait + return proc def win32_daemonize(command, cwd, stdout=None, stderr=None, append=True): diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index da33b76e5..5c8aa57ba 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -1,5 +1,6 @@ import atexit import heapq +import math import time from queue import Empty as QueueEmpty from queue import Queue @@ -113,10 +114,12 @@ class ProcessExecutionError(OSError): well as the command line used to create the process (``argv``) """ + # pylint: disable-next=super-init-not-called def __init__(self, argv, retcode, stdout, stderr, message=None, *, host=None): # we can't use 'super' here since OSError only keeps the first 2 args, # which leads to failuring in loading this object from a pickle.dumps. + # pylint: disable-next=non-parent-init-called Exception.__init__(self, argv, retcode, stdout, stderr) self.message = message @@ -310,7 +313,7 @@ def run_proc(proc, retcode, timeout=None): BY_POSITION = object() BY_TYPE = object() DEFAULT_ITER_LINES_MODE = BY_POSITION -DEFAULT_BUFFER_SIZE = _INFINITE = float("inf") +DEFAULT_BUFFER_SIZE = math.inf def iter_lines( @@ -376,7 +379,7 @@ def iter_lines( buffer = buffers[t] if buffer_size > 0: buffer.append(line) - if buffer_size < _INFINITE: + if buffer_size < math.inf: del buffer[:-buffer_size] if mode is BY_POSITION: diff --git a/plumbum/machines/local.py b/plumbum/machines/local.py index a9a4e5f41..a0fc36af2 100644 --- a/plumbum/machines/local.py +++ b/plumbum/machines/local.py @@ -344,7 +344,10 @@ def list_processes(self): # pylint: disable=no-self-use output = tasklist("/V", "/FO", "CSV") lines = output.splitlines() rows = csv.reader(lines) - header = next(rows) + try: + header = next(rows) + except StopIteration: + raise RuntimeError("tasklist must at least have header") from None imgidx = header.index("Image Name") pididx = header.index("PID") statidx = header.index("Status") diff --git a/plumbum/machines/paramiko_machine.py b/plumbum/machines/paramiko_machine.py index 1c5559eea..36f8185ab 100644 --- a/plumbum/machines/paramiko_machine.py +++ b/plumbum/machines/paramiko_machine.py @@ -449,6 +449,9 @@ def _path_stat(self, fn): res.text_mode = "regular file" return res + def daemonic_popen(self, command, cwd="/", stdout=None, stderr=None, append=True): + raise NotImplementedError("This is not implemented on ParamikoMachine!") + ################################################################################################### # Make paramiko.Channel adhere to the socket protocol, namely, send and recv should fail diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index 097652d51..e214850c7 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -349,7 +349,7 @@ def _path_glob(self, fn, pattern): # shquote does not work here due to the way bash loops use space as a separator pattern = pattern.replace(" ", r"\ ") fn = fn.replace(" ", r"\ ") - matches = self._session.run(fr"for fn in {fn}/{pattern}; do echo $fn; done")[ + matches = self._session.run(rf"for fn in {fn}/{pattern}; do echo $fn; done")[ 1 ].splitlines() if len(matches) == 1 and not self._path_stat(matches[0]): diff --git a/pyproject.toml b/pyproject.toml index 7f03dc928..5e679b4f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ all = true [tool.pylint] master.py-version = "3.6" master.jobs = "0" -master.fail-under = "9.97" reports.output-format = "colorized" similarities.ignore-imports = "yes" messages_control.enable = [ @@ -67,6 +66,7 @@ messages_control.disable = [ "arguments-differ", # TODO: investigate "attribute-defined-outside-init", # TODO: investigate "broad-except", # TODO: investigate + "consider-using-with", # TODO: should be handled "cyclic-import", "duplicate-code", # TODO: check "fixme", @@ -78,6 +78,7 @@ messages_control.disable = [ "missing-function-docstring", "missing-module-docstring", "no-member", + #"non-parent-init-called", # TODO: should be looked at "protected-access", "too-few-public-methods", "too-many-arguments", @@ -90,6 +91,5 @@ messages_control.disable = [ "too-many-public-methods", "too-many-return-statements", "too-many-statements", - "non-parent-init-called", # TODO: should be looked at "unidiomatic-typecheck", # TODO: might be able to remove ] From 96e996aab720298e724afb420b1e5179b469c167 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 10 Feb 2022 17:01:25 -0500 Subject: [PATCH 20/37] ci: fix version number missing in jobs (#591) * ci: fix version number missing in jobs * Update ci.yml --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fb2d7599..0da79ea78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: actions/setup-python@v2 - uses: pre-commit/action@v2.0.3 - name: pylint @@ -47,6 +49,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -96,6 +100,8 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Build SDist and wheel run: pipx run build - uses: actions/upload-artifact@v2 From d88ba3fbeea4488844e57bf33f83903d98505e96 Mon Sep 17 00:00:00 2001 From: Andrii Oriekhov Date: Mon, 7 Mar 2022 17:20:43 +0200 Subject: [PATCH 21/37] docs: add GitHub URL for PyPi (#593) * add GitHub URL for PyPi * Update setup.cfg * Update setup.cfg Co-authored-by: Henry Schreiner --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.cfg b/setup.cfg index c6f1770d2..34df273af 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,10 @@ keywords = execution, color, cli +project_urls = + Bug Tracker = https://github.com/tomerfiliba/plumbum/issues + Changelog = https://plumbum.readthedocs.io/en/latest/changelog.html + Source = https://github.com/tomerfiliba/plumbum provides = plumbum [options] From 723826e4644b84817e8a952bb087437e29f67637 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 28 Mar 2022 17:36:09 -0400 Subject: [PATCH 22/37] style: bump black (#595) * style: bump black Signed-off-by: Henry Schreiner * fix: pylint fixes, new_session kw-only --- .pre-commit-config.yaml | 10 +++++----- plumbum/cli/application.py | 2 +- plumbum/commands/base.py | 2 +- plumbum/machines/paramiko_machine.py | 2 +- plumbum/machines/remote.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a95da7295..8f3ef488f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/psf/black - rev: "22.1.0" + rev: "22.3.0" hooks: - id: black @@ -30,7 +30,7 @@ repos: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: "v2.31.0" + rev: "v2.31.1" hooks: - id: pyupgrade args: ["--py36-plus"] @@ -41,7 +41,7 @@ repos: - id: setup-cfg-fmt - repo: https://github.com/hadialqattan/pycln - rev: v1.1.0 + rev: v1.2.5 hooks: - id: pycln args: [--config=pyproject.toml] @@ -55,7 +55,7 @@ repos: additional_dependencies: [flake8-bugbear, flake8-print, flake8-2020] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.931" + rev: "v0.942" hooks: - id: mypy files: plumbum @@ -64,7 +64,7 @@ repos: # This wants the .mo files removed - repo: https://github.com/mgedmin/check-manifest - rev: "0.47" + rev: "0.48" hooks: - id: check-manifest stages: [manual] diff --git a/plumbum/cli/application.py b/plumbum/cli/application.py index f5e98393a..fa523031d 100644 --- a/plumbum/cli/application.py +++ b/plumbum/cli/application.py @@ -889,7 +889,7 @@ def switchs(by_groups, show_groups): lgrp = T_(grp) if grp in _switch_groups else grp print(self.COLOR_GROUP_TITLES[grp] | lgrp + ":") - for si in sorted(swinfos, key=lambda si: si.names): + for si in sorted(swinfos, key=lambda x: x.names): swnames = ", ".join( ("-" if len(n) == 1 else "--") + n for n in si.names diff --git a/plumbum/commands/base.py b/plumbum/commands/base.py index 5650ce3fd..51e667df8 100644 --- a/plumbum/commands/base.py +++ b/plumbum/commands/base.py @@ -566,7 +566,7 @@ def popen(self, args=(), **kwargs): class ConcreteCommand(BaseCommand): QUOTE_LEVEL: ClassVar[int] - __slots__ = ("executable", "custom_encoding") + __slots__ = ("executable",) def __init__(self, executable, encoding): self.executable = executable diff --git a/plumbum/machines/paramiko_machine.py b/plumbum/machines/paramiko_machine.py index 36f8185ab..a973d3c2e 100644 --- a/plumbum/machines/paramiko_machine.py +++ b/plumbum/machines/paramiko_machine.py @@ -287,7 +287,7 @@ def sftp(self): return self._sftp def session( - self, isatty=False, term="vt100", width=80, height=24, new_session=False + self, isatty=False, term="vt100", width=80, height=24, *, new_session=False ): # new_session is ignored for ParamikoMachine trans = self._client.get_transport() diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index e214850c7..8e0026342 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -100,7 +100,7 @@ def getdelta(self): class RemoteCommand(ConcreteCommand): - __slots__ = ["remote", "executable"] + __slots__ = ("remote",) QUOTE_LEVEL = 1 def __init__(self, remote, executable, encoding="auto"): @@ -276,7 +276,7 @@ def python(self): self._python = self["python3"] return self._python - def session(self, isatty=False, new_session=False): + def session(self, isatty=False, *, new_session=False): """Creates a new :class:`ShellSession ` object; this invokes the user's shell on the remote machine and executes commands on it over stdin/stdout/stderr""" raise NotImplementedError() From 5fa1452846aae4a1fd2904bf145b0801c2050508 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 30 Mar 2022 18:35:05 -0400 Subject: [PATCH 23/37] docs: update changelog (#596) --- .pre-commit-config.yaml | 4 ++-- CHANGELOG.rst | 5 ++++- plumbum/colorlib/factories.py | 4 ++-- plumbum/commands/processes.py | 3 ++- plumbum/machines/remote.py | 2 +- plumbum/path/utils.py | 3 ++- pyproject.toml | 17 ++++++++++++++--- 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f3ef488f..8ac80fed5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: rev: v1.2.5 hooks: - id: pycln - args: [--config=pyproject.toml] + args: [--all] stages: [manual] - repo: https://github.com/pycqa/flake8 @@ -59,7 +59,7 @@ repos: hooks: - id: mypy files: plumbum - args: [--show-error-codes] + args: [] additional_dependencies: [typed-ast, types-paramiko, types-setuptools] # This wants the .mo files removed diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 760613253..0e3a9e5e7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,11 @@ ----- * Drop Python 2.7 and 3.5 support (`#573 `_) +* Lots of extended checks and fixes for problems exposed. * Color: support ``NO_COLOR``/``FORCE_COLOR`` (`#575 `_) - +* Commands: New ``iter_lines`` ``buffer_size`` parameter (`#582 `_) +* Commands: cache remote commands (`#583 `_) +* Exceptions: fix for exception pickling (`#586 `_) 1.7.2 diff --git a/plumbum/colorlib/factories.py b/plumbum/colorlib/factories.py index 34cdd4e9c..5c5b6979f 100644 --- a/plumbum/colorlib/factories.py +++ b/plumbum/colorlib/factories.py @@ -1,11 +1,11 @@ """ Color-related factories. They produce Styles. - """ import sys from functools import reduce +from typing import Any from .names import color_names, default_styles from .styles import ColorNotFound @@ -108,7 +108,7 @@ def __enter__(self): """This will reset the color on leaving the with statement.""" return self - def __exit__(self, _type, _value, _traceback) -> None: + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: """This resets a FG/BG color or all styles, due to different definition of RESET for the factories.""" diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 5c8aa57ba..96647ba5b 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -262,7 +262,8 @@ def _shutdown_bg_threads(): global _shutting_down # pylint: disable=global-statement _shutting_down = True # Make sure this still exists (don't throw error in atexit!) - if _timeout_queue: + # TODO: not sure why this would be "falsy", though + if _timeout_queue: # type: ignore[truthy-bool] _timeout_queue.put((SystemExit, 0)) # grace period bgthd.join(0.1) diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index 8e0026342..9f18421d7 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -6,7 +6,7 @@ from plumbum.lib import ProcInfo from plumbum.machines.base import BaseMachine from plumbum.machines.env import BaseEnv -from plumbum.machines.local import LocalPath +from plumbum.path.local import LocalPath from plumbum.path.remote import RemotePath, RemoteWorkdir, StatRes diff --git a/plumbum/path/utils.py b/plumbum/path/utils.py index 558b93114..5f644fdd3 100644 --- a/plumbum/path/utils.py +++ b/plumbum/path/utils.py @@ -1,7 +1,8 @@ import os -from plumbum.machines.local import LocalPath, local +from plumbum.machines.local import local from plumbum.path.base import Path +from plumbum.path.local import LocalPath def delete(*paths): diff --git a/pyproject.toml b/pyproject.toml index 5e679b4f3..b34fa8962 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,20 @@ files = ["plumbum"] python_version = "3.6" warn_unused_configs = true warn_unused_ignores = true +show_error_codes = true +enable_error_code = ["ignore-without-code", "truthy-bool"] +disallow_any_generics = false +disallow_subclassing_any = false +disallow_untyped_calls = false +disallow_untyped_defs = false +disallow_incomplete_defs = true +check_untyped_defs = false +disallow_untyped_decorators = false +no_implicit_optional = true +warn_redundant_casts = true +warn_return_any = false +no_implicit_reexport = true +strict_equality = true [[tool.mypy.overrides]] @@ -50,9 +64,6 @@ ignore = [ "CONTRIBUTING.rst", ] -[tool.pycln] -all = true - [tool.pylint] master.py-version = "3.6" From 850e663692b91bd5e2fe2e1b4f02130ba99dcc09 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 24 Jun 2022 11:26:23 -0400 Subject: [PATCH 24/37] chore: pylint 2.14 support, pin pylint (#606) Signed-off-by: Henry Schreiner --- noxfile.py | 2 +- plumbum/cli/progress.py | 2 +- plumbum/fs/atomic.py | 2 +- pyproject.toml | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 9b68bfe7a..9fae464b4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -22,7 +22,7 @@ def pylint(session): Run pylint. """ - session.install(".", "paramiko", "ipython", "pylint") + session.install(".", "paramiko", "ipython", "pylint~=2.14.3") session.run("pylint", "plumbum", *session.posargs) diff --git a/plumbum/cli/progress.py b/plumbum/cli/progress.py index 3d21a8a61..fce6b3cab 100644 --- a/plumbum/cli/progress.py +++ b/plumbum/cli/progress.py @@ -69,7 +69,7 @@ def __next__(self): return rval def next(self): - return self.__next__() + return next(self) @property def value(self): diff --git a/plumbum/fs/atomic.py b/plumbum/fs/atomic.py index 5aebc80fa..d796b2f9c 100644 --- a/plumbum/fs/atomic.py +++ b/plumbum/fs/atomic.py @@ -290,7 +290,7 @@ def acquire(self): return self._ctx = self.atomicfile.locked(blocking=False) try: - self._ctx.__enter__() + self._ctx.__enter__() # pylint: disable=unnecessary-dunder-call except OSError: self._ctx = None try: diff --git a/pyproject.toml b/pyproject.toml index b34fa8962..27ddb9908 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ ignore = [ [tool.pylint] master.py-version = "3.6" master.jobs = "0" +master.load-plugins = ["pylint.extensions.no_self_use"] reports.output-format = "colorized" similarities.ignore-imports = "yes" messages_control.enable = [ @@ -103,4 +104,5 @@ messages_control.disable = [ "too-many-return-statements", "too-many-statements", "unidiomatic-typecheck", # TODO: might be able to remove + "unnecessary-lambda-assignment", # TODO: 4 instances ] From 3d27e6aa19650e6be057cb13bd9766790e17ec15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 14:29:59 -0400 Subject: [PATCH 25/37] chore(deps): bump pre-commit/action from 2.0.3 to 3.0.0 (#601) Bumps [pre-commit/action](https://github.com/pre-commit/action) from 2.0.3 to 3.0.0. - [Release notes](https://github.com/pre-commit/action/releases) - [Commits](https://github.com/pre-commit/action/compare/v2.0.3...v3.0.0) --- updated-dependencies: - dependency-name: pre-commit/action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0da79ea78..03c47496b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: with: fetch-depth: 0 - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.3 + - uses: pre-commit/action@v3.0.0 - name: pylint run: | echo "::add-matcher::$GITHUB_WORKSPACE/.github/matchers/pylint.json" From 16113fdcfc4f3e7164ae7e726e90b0594de64e51 Mon Sep 17 00:00:00 2001 From: Jesse London Date: Fri, 24 Jun 2022 17:36:36 -0500 Subject: [PATCH 26/37] fix: made StdinDataRedirection compatible with modifiers writing to stdout (#605) * made StdinDataRedirection compatible with modifiers writing to stdout _I.e._ fixed: (cat << "meow") & FG Modifiers writing to stdout -- _e.g._: FG, TEE, RETCODE(FG=True) and TF(FG=True) -- invoke commands with the specification `stdin=None`. This is handled correctly by BaseCommand; however, StdinDataRedirection misinterprets this as a further attempt to specify stdin (_i.e._ to specify _something_ rather than _nothing_). This change brings StdinDataRedirection in line with its base class, ignoring `None` as well as `PIPE`. (Though a technical detail, the class would also have failed any invocation setting `stdin=PIPE`, along the lines of: "TypeError: got multiple values for keyword argument 'stdin'". The kwargs passed to the wrapped command's `popen` are corrected as well.) resolves #604 * style: run pre-commit Signed-off-by: Henry Schreiner Co-authored-by: Henry Schreiner --- plumbum/commands/base.py | 5 +++-- tests/test_local.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/plumbum/commands/base.py b/plumbum/commands/base.py index 51e667df8..eafbde37c 100644 --- a/plumbum/commands/base.py +++ b/plumbum/commands/base.py @@ -547,7 +547,7 @@ def machine(self): return self.cmd.machine def popen(self, args=(), **kwargs): - if "stdin" in kwargs and kwargs["stdin"] != PIPE: + if kwargs.get("stdin") not in (PIPE, None): raise RedirectionError("stdin is already redirected") data = self.data if isinstance(data, str) and self._get_encoding() is not None: @@ -558,8 +558,9 @@ def popen(self, args=(), **kwargs): f.write(chunk) data = data[self.CHUNK_SIZE :] f.seek(0) + kwargs["stdin"] = f # try: - return self.cmd.popen(args, stdin=f, **kwargs) + return self.cmd.popen(args, **kwargs) # finally: # f.close() diff --git a/tests/test_local.py b/tests/test_local.py index dc76a7bc3..d96d39b49 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -639,6 +639,25 @@ def test_tee_race(self, capfd): assert result[1] == EXPECT assert EXPECT == capfd.readouterr()[0] + @skip_on_windows + @pytest.mark.parametrize( + "modifier, expected", + [ + (FG, None), + (TF(FG=True), True), + (RETCODE(FG=True), 0), + (TEE, (0, "meow", "")), + ], + ) + def test_redirection_stdin_modifiers_fg(self, modifier, expected, capfd): + "StdinDataRedirection compatible with modifiers which write to stdout" + from plumbum.cmd import cat + + cmd = cat << "meow" + + assert cmd & modifier == expected + assert capfd.readouterr() == ("meow", "") + @skip_on_windows def test_logger_pipe(self): from plumbum.cmd import bash From 1a31029b7ea2d89c5a2e2d67583455aa315d16dc Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 29 Jun 2022 02:02:17 -0400 Subject: [PATCH 27/37] chore: simpler dependabot (#609) Ignores no longer needed after April 2022. Dependabot keeps the same style pinning now. --- .github/dependabot.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f08ed35a5..2c7d17083 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,5 +5,3 @@ updates: directory: "/" schedule: interval: "daily" - ignore: - - dependency-name: "actions/*" From 566d91e5078171360251b6b913c203bc5a009d89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:21:59 -0400 Subject: [PATCH 28/37] chore(deps): bump actions/download-artifact from 2 to 3 (#613) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 2 to 3. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03c47496b..8a6c8c2d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: artifact path: dist From 5d4994cf2d4ed70f6690aba7cda698cfef25da2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:22:11 -0400 Subject: [PATCH 29/37] chore(deps): bump actions/cache from 2 to 3 (#611) Bumps [actions/cache](https://github.com/actions/cache) from 2 to 3. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a6c8c2d5..72554d413 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: runner.os == 'Linux' && startsWith(matrix.python-version, 'pypy') with: path: ~/.cache/pip From f60d460e5ef019a01bb0638741cf8ca30bf190d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:22:29 -0400 Subject: [PATCH 30/37] chore(deps): bump actions/checkout from 2 to 3 (#612) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72554d413..e94790c2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: name: Format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - uses: actions/setup-python@v2 @@ -48,7 +48,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -99,7 +99,7 @@ jobs: name: Dist runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build SDist and wheel From 4bfe7da14862d78f6fe0eb2aa2444b5941ba949c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 16:19:19 -0400 Subject: [PATCH 31/37] chore(deps): bump actions/setup-python from 2 to 4 (#614) * chore(deps): bump actions/setup-python from 2 to 4 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Apply suggestions from code review Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e94790c2b..f18da4cef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,9 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" - uses: pre-commit/action@v3.0.0 - name: pylint run: | @@ -53,7 +55,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -129,7 +131,9 @@ jobs: needs: [tests] runs-on: ubuntu-20.04 steps: - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" - name: Install coveralls run: pip install coveralls - name: Coveralls Finished From 70380ddc5ad346ab6758506f5000864260a9c418 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 16:21:26 -0400 Subject: [PATCH 32/37] chore(deps): bump actions/upload-artifact from 2 to 3 (#610) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 2 to 3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f18da4cef..3f3689ff1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: fetch-depth: 0 - name: Build SDist and wheel run: pipx run build - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: path: dist/* From e34d9e1f1492efd1f5e8f3d3f1270efe49cf986c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Aug 2022 17:46:41 -0400 Subject: [PATCH 33/37] chore(deps): bump pypa/gh-action-pypi-publish from 1.5.0 to 1.5.1 (#615) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f3689ff1..f65c073ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,7 @@ jobs: name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@v1.5.0 + - uses: pypa/gh-action-pypi-publish@v1.5.1 with: user: __token__ password: ${{ secrets.pypi_password }} From c80c64bb73a8512bed49720490b9fc57f61cd058 Mon Sep 17 00:00:00 2001 From: Ganden Schaffner Date: Mon, 3 Oct 2022 13:46:31 -0700 Subject: [PATCH 34/37] feat: support SSH tunnels with dynamic free port allocation (#608) * Add reverse SSH tunnel remote port detection and SshTunnel properties * Test reverse SSH tunnel remote port detection and SshTunnel properties * Add automatic local port allocation for SSH tunnels * Test automatic local port allocation for SSH tunnels --- plumbum/machines/session.py | 3 +- plumbum/machines/ssh_machine.py | 48 +++++++++++++++++++++++-- tests/test_remote.py | 64 +++++++++++++++++++++++++-------- 3 files changed, 96 insertions(+), 19 deletions(-) diff --git a/plumbum/machines/session.py b/plumbum/machines/session.py index f292ededc..3ebba97e6 100644 --- a/plumbum/machines/session.py +++ b/plumbum/machines/session.py @@ -217,6 +217,7 @@ def __init__( self.isatty = isatty self._lock = threading.RLock() self._current = None + self._startup_result = None if connect_timeout: def closer(): @@ -228,7 +229,7 @@ def closer(): timer = threading.Timer(connect_timeout, closer) timer.start() try: - self.run("") + self._startup_result = self.run("") finally: if connect_timeout: timer.cancel() diff --git a/plumbum/machines/ssh_machine.py b/plumbum/machines/ssh_machine.py index 648b0efbf..79bad6b5f 100644 --- a/plumbum/machines/ssh_machine.py +++ b/plumbum/machines/ssh_machine.py @@ -1,4 +1,7 @@ +import re +import socket import warnings +from contextlib import closing from plumbum.commands import ProcessExecutionError, shquote from plumbum.lib import IS_WIN32 @@ -9,14 +12,33 @@ from plumbum.path.remote import RemotePath +def _get_free_port(): + """Attempts to find a free port.""" + s = socket.socket() + with closing(s): + s.bind(("localhost", 0)) + return s.getsockname()[1] + + class SshTunnel: """An object representing an SSH tunnel (created by :func:`SshMachine.tunnel `)""" - __slots__ = ["_session", "__weakref__"] + __slots__ = ["_session", "_lport", "_dport", "_reverse", "__weakref__"] - def __init__(self, session): + def __init__(self, session, lport, dport, reverse): self._session = session + self._lport = lport + self._dport = dport + self._reverse = reverse + if reverse and str(dport) == "0" and session._startup_result is not None: + # Try to detect assigned remote port. + regex = re.compile( + r"^Allocated port (\d+) for remote forward to .+$", re.MULTILINE + ) + match = regex.search(session._startup_result[2]) + if match: + self._dport = match.group(1) def __repr__(self): tunnel = self._session.proc if self._session.alive() else "(defunct)" @@ -32,6 +54,21 @@ def close(self): """Closes(terminates) the tunnel""" self._session.close() + @property + def lport(self): + """Tunneled port or socket on the local machine.""" + return self._lport + + @property + def dport(self): + """Tunneled port or socket on the remote machine.""" + return self._dport + + @property + def reverse(self): + """Represents if the tunnel is a reverse tunnel.""" + return self._reverse + class SshMachine(BaseRemoteMachine): """ @@ -272,6 +309,8 @@ def tunnel( """ formatted_lhost = "" if lhost is None else f"[{lhost}]:" formatted_dhost = "" if dhost is None else f"[{dhost}]:" + if str(lport) == "0": + lport = _get_free_port() ssh_opts = ( [ "-L", @@ -287,7 +326,10 @@ def tunnel( return SshTunnel( ShellSession( proc, self.custom_encoding, connect_timeout=self.connect_timeout - ) + ), + lport, + dport, + reverse, ) def _translate_drive_letter(self, path): # pylint: disable=no-self-use diff --git a/tests/test_remote.py b/tests/test_remote.py index 19245daa5..30d14d0ea 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -36,6 +36,10 @@ def strassert(one, two): assert str(one) == str(two) +def assert_is_port(port): + assert 0 < int(port) < 2**16 + + # TEST_HOST = "192.168.1.143" TEST_HOST = "127.0.0.1" if TEST_HOST not in ("::1", "127.0.0.1", "localhost"): @@ -444,9 +448,9 @@ def test_touch(self): rfile.delete() -def serve_reverse_tunnel(queue): +def serve_reverse_tunnel(queue, port): s = socket.socket() - s.bind(("", 12223)) + s.bind(("", port)) s.listen(1) s2, _ = s.accept() data = s2.recv(100).decode("ascii").strip() @@ -460,7 +464,8 @@ class TestRemoteMachine(BaseRemoteMachineTest): def _connect(self): return SshMachine(TEST_HOST) - def test_tunnel(self): + @pytest.mark.parametrize("dynamic_lport", [False, True]) + def test_tunnel(self, dynamic_lport): for tunnel_prog in (self.TUNNEL_PROG_AF_INET, self.TUNNEL_PROG_AF_UNIX): with self._connect() as rem: @@ -472,9 +477,21 @@ def test_tunnel(self): except ValueError: dhost = None - with rem.tunnel(12222, port_or_socket, dhost=dhost): + if not dynamic_lport: + lport = 12222 + else: + lport = 0 + + with rem.tunnel(lport, port_or_socket, dhost=dhost) as tun: + if not dynamic_lport: + assert tun.lport == lport + else: + assert_is_port(tun.lport) + assert tun.dport == port_or_socket + assert not tun.reverse + s = socket.socket() - s.connect(("localhost", 12222)) + s.connect(("localhost", tun.lport)) s.send(b"world") data = s.recv(100) s.close() @@ -482,10 +499,19 @@ def test_tunnel(self): print(p.communicate()) assert data == b"hello world" - def test_reverse_tunnel(self): + @pytest.mark.parametrize("dynamic_dport", [False, True]) + def test_reverse_tunnel(self, dynamic_dport): + lport = 12223 + dynamic_dport with self._connect() as rem: - get_unbound_socket_remote = """import sys, socket + queue = Queue() + tunnel_server = Thread(target=serve_reverse_tunnel, args=(queue, lport)) + tunnel_server.start() + message = str(time.time()) + + if not dynamic_dport: + + get_unbound_socket_remote = """import sys, socket s = socket.socket() s.bind(("", 0)) s.listen(1) @@ -493,20 +519,28 @@ def test_reverse_tunnel(self): sys.stdout.flush() s.close() """ - p = (rem.python["-u"] << get_unbound_socket_remote).popen() - remote_socket = p.stdout.readline().decode("ascii").strip() - queue = Queue() - tunnel_server = Thread(target=serve_reverse_tunnel, args=(queue,)) - tunnel_server.start() - message = str(time.time()) - with rem.tunnel(12223, remote_socket, dhost="localhost", reverse=True): + p = (rem.python["-u"] << get_unbound_socket_remote).popen() + remote_socket = p.stdout.readline().decode("ascii").strip() + else: + remote_socket = 0 + + with rem.tunnel( + lport, remote_socket, dhost="localhost", reverse=True + ) as tun: + assert tun.lport == lport + if not dynamic_dport: + assert tun.dport == remote_socket + else: + assert_is_port(tun.dport) + assert tun.reverse + remote_send_af_inet = """import socket s = socket.socket() s.connect(("localhost", {})) s.send("{}".encode("ascii")) s.close() """.format( - remote_socket, message + tun.dport, message ) (rem.python["-u"] << remote_send_af_inet).popen() tunnel_server.join(timeout=1) From 42d661d154f6abcd028e66635fc245cebe316355 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 3 Oct 2022 17:51:08 -0400 Subject: [PATCH 35/37] fix: abc & a few style issues (#617) * fix: ABC was applied to the wrong class Signed-off-by: Henry Schreiner * chore: update style and fix a few issues Signed-off-by: Henry Schreiner * fix: ABCMeta instead of ABC to avoid weakref on 3.6 issue Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- .pre-commit-config.yaml | 17 +++++++++-------- noxfile.py | 4 ++-- plumbum/cli/image.py | 21 +++++++++++---------- plumbum/cli/progress.py | 13 +++++++------ plumbum/colorlib/__init__.py | 2 +- plumbum/colorlib/styles.py | 6 +++--- plumbum/commands/processes.py | 4 ++-- plumbum/fs/atomic.py | 4 +++- plumbum/path/base.py | 3 +-- plumbum/path/local.py | 11 +++++++---- plumbum/path/remote.py | 10 ++++++---- plumbum/typed_env.py | 2 +- setup.cfg | 10 ++++++++-- tests/test_validate.py | 10 +++++----- 14 files changed, 66 insertions(+), 51 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ac80fed5..3f5f487cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.3.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -20,7 +20,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/psf/black - rev: "22.3.0" + rev: "22.8.0" hooks: - id: black @@ -30,32 +30,33 @@ repos: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: "v2.31.1" + rev: "v2.38.2" hooks: - id: pyupgrade args: ["--py36-plus"] - repo: https://github.com/asottile/setup-cfg-fmt - rev: "v1.20.0" + rev: "v2.0.0" hooks: - id: setup-cfg-fmt + args: [--include-version-classifiers, --max-py-version=3.11] - repo: https://github.com/hadialqattan/pycln - rev: v1.2.5 + rev: v2.1.1 hooks: - id: pycln args: [--all] stages: [manual] - repo: https://github.com/pycqa/flake8 - rev: "4.0.1" + rev: "5.0.4" hooks: - id: flake8 exclude: docs/conf.py additional_dependencies: [flake8-bugbear, flake8-print, flake8-2020] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.942" + rev: "v0.981" hooks: - id: mypy files: plumbum @@ -70,7 +71,7 @@ repos: stages: [manual] - repo: https://github.com/codespell-project/codespell - rev: v2.1.0 + rev: v2.2.1 hooks: - id: codespell diff --git a/noxfile.py b/noxfile.py index 9fae464b4..d6445c2cd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -47,10 +47,10 @@ def docs(session): if session.posargs: if "serve" in session.posargs: - print("Launching docs at http://localhost:8000/ - use Ctrl-C to quit") + session.log("Launching docs at http://localhost:8000/ - use Ctrl-C to quit") session.run("python", "-m", "http.server", "8000", "-d", "_build/html") else: - print("Unsupported argument to docs") + session.log("Unsupported argument to docs") @nox.session diff --git a/plumbum/cli/image.py b/plumbum/cli/image.py index 293b606d2..08c6d8254 100644 --- a/plumbum/cli/image.py +++ b/plumbum/cli/image.py @@ -1,3 +1,5 @@ +import sys + from plumbum import colors from .. import cli @@ -56,9 +58,10 @@ def show_pil(self, im): for y in range(size[1]): for x in range(size[0] - 1): pix = new_im.getpixel((x, y)) - print(colors.bg.rgb(*pix), " ", sep="", end="") # '\u2588' - print(colors.reset, " ", sep="") - print(colors.reset) + sys.stdout.write(colors.bg.rgb(*pix) + " ") # '\u2588' + sys.stdout.write(colors.reset + " \n") + sys.stdout.write(colors.reset + "\n") + sys.stdout.flush() def show_pil_double(self, im): "Show double resolution on some fonts" @@ -71,14 +74,12 @@ def show_pil_double(self, im): for x in range(size[0] - 1): pix = new_im.getpixel((x, y * 2)) pixl = new_im.getpixel((x, y * 2 + 1)) - print( - colors.bg.rgb(*pixl) & colors.fg.rgb(*pix), - "\u2580", - sep="", - end="", + sys.stdout.write( + (colors.bg.rgb(*pixl) & colors.fg.rgb(*pix)) + "\u2580" ) - print(colors.reset, " ", sep="") - print(colors.reset) + sys.stdout.write(colors.reset + " \n") + sys.stdout.write(colors.reset + "\n") + sys.stdout.flush() class ShowImageApp(cli.Application): diff --git a/plumbum/cli/progress.py b/plumbum/cli/progress.py index fce6b3cab..85513d154 100644 --- a/plumbum/cli/progress.py +++ b/plumbum/cli/progress.py @@ -136,9 +136,10 @@ def done(self): self.value = self.length self.display() if self.clear and not self.has_output: - print("\r", len(str(self)) * " ", "\r", end="", sep="") + sys.stdout.write("\r" + len(str(self)) * " " + "\r") else: - print() + sys.stdout.write("\n") + sys.stdout.flush() def __str__(self): width = get_terminal_size(default=(0, 0))[0] @@ -175,11 +176,11 @@ def __str__(self): def display(self): disptxt = str(self) if self.width == 0 or self.has_output: - print(disptxt) + sys.stdout.write(disptxt + "\n") else: - print("\r", end="") - print(disptxt, end="") - sys.stdout.flush() + sys.stdout.write("\r") + sys.stdout.write(disptxt) + sys.stdout.flush() class ProgressIPy(ProgressBase): # pragma: no cover diff --git a/plumbum/colorlib/__init__.py b/plumbum/colorlib/__init__.py index e80bd10b7..12b532428 100644 --- a/plumbum/colorlib/__init__.py +++ b/plumbum/colorlib/__init__.py @@ -29,7 +29,7 @@ def load_ipython_extension(ipython): # pragma: no cover try: from ._ipython_ext import OutputMagics # pylint:disable=import-outside-toplevel except ImportError: - print("IPython required for the IPython extension to be loaded.") + print("IPython required for the IPython extension to be loaded.") # noqa: T201 raise ipython.push({"colors": htmlcolors}) diff --git a/plumbum/colorlib/styles.py b/plumbum/colorlib/styles.py index afb842dfc..4dcdf90ab 100644 --- a/plumbum/colorlib/styles.py +++ b/plumbum/colorlib/styles.py @@ -12,7 +12,7 @@ import platform import re import sys -from abc import ABC, abstractmethod +from abc import ABCMeta, abstractmethod from copy import copy from typing import IO, Dict, Optional, Union @@ -74,7 +74,7 @@ class ResetNotSupported(Exception): for this Style.""" -class Color(ABC): +class Color: """\ Loaded with ``(r, g, b, fg)`` or ``(color, fg=fg)``. The second signature is a short cut and will try full and hex loading. @@ -331,7 +331,7 @@ def limit_representation(self, val): return self if self.representation <= val else self.to_representation(val) -class Style: +class Style(metaclass=ABCMeta): """This class allows the color changes to be called directly to write them to stdout, ``[]`` calls to wrap colors (or the ``.wrap`` method) and can be called in a with statement. diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 96647ba5b..51d930c2f 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -262,7 +262,7 @@ def _shutdown_bg_threads(): global _shutting_down # pylint: disable=global-statement _shutting_down = True # Make sure this still exists (don't throw error in atexit!) - # TODO: not sure why this would be "falsy", though + # TODO: not sure why this would be "falsey", though if _timeout_queue: # type: ignore[truthy-bool] _timeout_queue.put((SystemExit, 0)) # grace period @@ -367,7 +367,7 @@ def iter_lines( assert mode in (BY_POSITION, BY_TYPE) encoding = getattr(proc, "custom_encoding", None) or "utf-8" - decode = lambda s: s.decode(encoding, errors="replace").rstrip() + decode = lambda s: s.decode(encoding, errors="replace").rstrip() # noqa: E731 _register_proc_timeout(proc, timeout) diff --git a/plumbum/fs/atomic.py b/plumbum/fs/atomic.py index d796b2f9c..1285998e0 100644 --- a/plumbum/fs/atomic.py +++ b/plumbum/fs/atomic.py @@ -19,7 +19,9 @@ from win32con import LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY from win32file import OVERLAPPED, LockFileEx, UnlockFile except ImportError: - print("On Windows, Plumbum requires Python for Windows Extensions (pywin32)") + print( # noqa: T201 + "On Windows, Plumbum requires Python for Windows Extensions (pywin32)" + ) raise @contextmanager diff --git a/plumbum/path/base.py b/plumbum/path/base.py index a1a93aaea..5f63b35f6 100644 --- a/plumbum/path/base.py +++ b/plumbum/path/base.py @@ -423,9 +423,8 @@ def resolve(self, strict=False): # pylint:disable=unused-argument @property def parents(self): """Pathlib like sequence of ancestors""" - join = lambda x, y: self._form(x) / y as_list = ( - reduce(join, self.parts[:i], self.parts[0]) + reduce(lambda x, y: self._form(x) / y, self.parts[:i], self.parts[0]) for i in range(len(self.parts) - 1, 0, -1) ) return tuple(as_list) diff --git a/plumbum/path/local.py b/plumbum/path/local.py index bdaab3c06..fe0e507bc 100644 --- a/plumbum/path/local.py +++ b/plumbum/path/local.py @@ -147,10 +147,13 @@ def with_suffix(self, suffix, depth=1): return LocalPath(self.dirname) / (name + suffix) def glob(self, pattern): - fn = lambda pat: [ - LocalPath(m) for m in glob.glob(os.path.join(glob.escape(str(self)), pat)) - ] - return self._glob(pattern, fn) + return self._glob( + pattern, + lambda pat: [ + LocalPath(m) + for m in glob.glob(os.path.join(glob.escape(str(self)), pat)) + ], + ) def delete(self): if not self.exists(): diff --git a/plumbum/path/remote.py b/plumbum/path/remote.py index 0741e2cf9..b17d7e6fb 100644 --- a/plumbum/path/remote.py +++ b/plumbum/path/remote.py @@ -173,10 +173,12 @@ def with_suffix(self, suffix, depth=1): return self.__class__(self.remote, self.dirname) / (name + suffix) def glob(self, pattern): - fn = lambda pat: [ - RemotePath(self.remote, m) for m in self.remote._path_glob(self, pat) - ] - return self._glob(pattern, fn) + return self._glob( + pattern, + lambda pat: [ + RemotePath(self.remote, m) for m in self.remote._path_glob(self, pat) + ], + ) def delete(self): if not self.exists(): diff --git a/plumbum/typed_env.py b/plumbum/typed_env.py index 4ad15d501..fe88aae2d 100644 --- a/plumbum/typed_env.py +++ b/plumbum/typed_env.py @@ -96,7 +96,7 @@ class CSV(_BaseVar): def __init__( self, name, default=NO_DEFAULT, type=str, separator="," ): # pylint:disable=redefined-builtin - super(TypedEnv.CSV, self).__init__(name, default=default) + super().__init__(name, default=default) self.type = type self.separator = separator diff --git a/setup.cfg b/setup.cfg index 34df273af..061c80c66 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Software Development :: Build Tools Topic :: System :: Systems Administration keywords = @@ -88,8 +89,13 @@ exclude_lines = [flake8] max-complexity = 50 -extend-ignore = E203, E501, E722, B950, E731 -select = C,E,F,W,B,B9 +extend-ignore = E203, E501, B950, T202 +extend-select = B9 +per-file-ignores = + tests/*: T + examples/*: T + experiments/*: T + plumbum/cli/application.py: T [codespell] ignore-words-list = ans,switchs,hart,ot,twoo,fo diff --git a/tests/test_validate.py b/tests/test_validate.py index 15d098895..a39b8cabd 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -5,7 +5,7 @@ class TestValidator: def test_named(self): class Try: @cli.positional(x=abs, y=str) - def main(selfy, x, y): + def main(selfy, x, y): # noqa: B902 pass assert Try.main.positional == [abs, str] @@ -14,7 +14,7 @@ def main(selfy, x, y): def test_position(self): class Try: @cli.positional(abs, str) - def main(selfy, x, y): + def main(selfy, x, y): # noqa: B902 pass assert Try.main.positional == [abs, str] @@ -23,7 +23,7 @@ def main(selfy, x, y): def test_mix(self): class Try: @cli.positional(abs, str, d=bool) - def main(selfy, x, y, z, d): + def main(selfy, x, y, z, d): # noqa: B902 pass assert Try.main.positional == [abs, str, None, bool] @@ -32,7 +32,7 @@ def main(selfy, x, y, z, d): def test_var(self): class Try: @cli.positional(abs, str, int) - def main(selfy, x, y, *g): + def main(selfy, x, y, *g): # noqa: B902 pass assert Try.main.positional == [abs, str] @@ -41,7 +41,7 @@ def main(selfy, x, y, *g): def test_defaults(self): class Try: @cli.positional(abs, str) - def main(selfy, x, y="hello"): + def main(selfy, x, y="hello"): # noqa: B902 pass assert Try.main.positional == [abs, str] From 0d8afef6825e7f7c730d1f758809f30b1ea52264 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 5 Oct 2022 12:36:30 -0400 Subject: [PATCH 36/37] chore: minor cleanups from reverb (#618) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- plumbum/cli/application.py | 12 ++++++------ plumbum/cli/config.py | 5 ++--- plumbum/cli/switches.py | 5 ++--- plumbum/cli/terminal.py | 20 +++++++++----------- plumbum/cli/termsize.py | 5 ++--- plumbum/colorlib/factories.py | 5 +++-- plumbum/colorlib/styles.py | 13 +++++-------- plumbum/commands/base.py | 10 ++++------ plumbum/commands/daemons.py | 5 ++--- plumbum/commands/processes.py | 9 +++------ plumbum/fs/atomic.py | 14 ++++++-------- plumbum/machines/local.py | 8 +++----- plumbum/machines/paramiko_machine.py | 5 ++--- plumbum/machines/remote.py | 11 ++++------- plumbum/machines/session.py | 19 ++++++------------- plumbum/path/base.py | 2 +- 16 files changed, 60 insertions(+), 88 deletions(-) diff --git a/plumbum/cli/application.py b/plumbum/cli/application.py index fa523031d..788ccb57a 100644 --- a/plumbum/cli/application.py +++ b/plumbum/cli/application.py @@ -723,7 +723,7 @@ def cleanup(self, retcode): def helpall(self): """Prints help messages of all sub-commands and quits""" self.help() - print("") + print() if self._subcommands: for name, subcls in sorted(self._subcommands.items()): @@ -744,7 +744,7 @@ def help(self): # @ReservedAssignment """Prints this help message and quits""" if self._get_prog_version(): self.version() - print("") + print() if self.DESCRIPTION: print(self.DESCRIPTION.strip() + "\n") @@ -860,7 +860,7 @@ def wrapped_paragraphs(text, width): tailargs.append(f"{m.varargs}...") tailargs = " ".join(tailargs) - utc = self.COLOR_USAGE_TITLE if self.COLOR_USAGE_TITLE else self.COLOR_USAGE + utc = self.COLOR_USAGE_TITLE or self.COLOR_USAGE print(utc | T_("Usage:")) with self.COLOR_USAGE: @@ -908,7 +908,7 @@ def switchs(by_groups, show_groups): yield si, prefix, self.COLOR_GROUPS[grp] if show_groups: - print("") + print() sw_width = ( max(len(prefix) for si, prefix, color in switchs(by_groups, False)) + 4 @@ -954,14 +954,14 @@ def switchs(by_groups, show_groups): for name, subcls in sorted(self._subcommands.items()): with gc: subapp = subcls.get() - doc = subapp.DESCRIPTION if subapp.DESCRIPTION else getdoc(subapp) + doc = subapp.DESCRIPTION or getdoc(subapp) if self.SUBCOMMAND_HELPMSG: help_str = doc + "; " if doc else "" help_str += self.SUBCOMMAND_HELPMSG.format( parent=self.PROGNAME, sub=name ) else: - help_str = doc if doc else "" + help_str = doc or "" msg = indentation.join( wrapper.wrap( diff --git a/plumbum/cli/config.py b/plumbum/cli/config.py index 75ccf5ff1..b75e5515b 100644 --- a/plumbum/cli/config.py +++ b/plumbum/cli/config.py @@ -1,3 +1,4 @@ +import contextlib from abc import ABC, abstractmethod from configparser import ConfigParser, NoOptionError, NoSectionError @@ -26,10 +27,8 @@ def __init__(self, filename): self.changed = False def __enter__(self): - try: + with contextlib.suppress(FileNotFoundError): self.read() - except FileNotFoundError: - pass return self def __exit__(self, exc_type, exc_val, exc_tb): diff --git a/plumbum/cli/switches.py b/plumbum/cli/switches.py index 0a9fcfd92..830e0008c 100644 --- a/plumbum/cli/switches.py +++ b/plumbum/cli/switches.py @@ -1,3 +1,4 @@ +import contextlib import inspect from abc import ABC, abstractmethod @@ -484,10 +485,8 @@ def __call__(self, value, check_csv=True): if opt == value: return opt # always return original value continue - try: + with contextlib.suppress(ValueError): return opt(value) - except ValueError: - pass raise ValueError(f"Invalid value: {value} (Expected one of {self.values})") def choices(self, partial=""): diff --git a/plumbum/cli/terminal.py b/plumbum/cli/terminal.py index 5b8e6d05e..5c506d699 100644 --- a/plumbum/cli/terminal.py +++ b/plumbum/cli/terminal.py @@ -4,15 +4,17 @@ """ +import contextlib import os import sys +from typing import List, Optional from plumbum import local from .progress import Progress from .termsize import get_terminal_size -__all__ = ( +__all__ = [ "readline", "ask", "choose", @@ -20,21 +22,21 @@ "get_terminal_size", "Progress", "get_terminal_size", -) +] -def __dir__(): +def __dir__() -> List[str]: return __all__ -def readline(message=""): +def readline(message: str = "") -> str: """Gets a line of input from the user (stdin)""" sys.stdout.write(message) sys.stdout.flush() return sys.stdin.readline() -def ask(question, default=None): +def ask(question: str, default: Optional[bool] = None) -> bool: """ Presents the user with a yes/no question. @@ -234,13 +236,9 @@ def pager(rows, pagercmd=None): # pragma: no cover pg.stdin.close() pg.wait() finally: - try: + with contextlib.suppress(Exception): rows.close() - except Exception: - pass if pg and pg.poll() is None: - try: + with contextlib.suppress(Exception): pg.terminate() - except Exception: - pass os.system("reset") diff --git a/plumbum/cli/termsize.py b/plumbum/cli/termsize.py index d0341b86e..d56e56918 100644 --- a/plumbum/cli/termsize.py +++ b/plumbum/cli/termsize.py @@ -3,6 +3,7 @@ --------------------- """ +import contextlib import os import platform import warnings @@ -87,12 +88,10 @@ def _ioctl_GWINSZ(fd: int) -> Optional[Tuple[int, int]]: def _get_terminal_size_linux() -> Optional[Tuple[int, int]]: cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) if not cr: - try: + with contextlib.suppress(Exception): fd = os.open(os.ctermid(), os.O_RDONLY) cr = _ioctl_GWINSZ(fd) os.close(fd) - except Exception: - pass if not cr: try: cr = (int(os.environ["LINES"]), int(os.environ["COLUMNS"])) diff --git a/plumbum/colorlib/factories.py b/plumbum/colorlib/factories.py index 5c5b6979f..c34e88194 100644 --- a/plumbum/colorlib/factories.py +++ b/plumbum/colorlib/factories.py @@ -3,8 +3,9 @@ """ +import functools +import operator import sys -from functools import reduce from typing import Any from .names import color_names, default_styles @@ -182,7 +183,7 @@ def get_colors_from_string(self, color=""): prev = self if styleslist: - prev = reduce(lambda a, b: a & b, styleslist) + prev = functools.reduce(operator.and_, styleslist) return prev if isinstance(prev, self._style) else prev.reset diff --git a/plumbum/colorlib/styles.py b/plumbum/colorlib/styles.py index 4dcdf90ab..4e8b276dc 100644 --- a/plumbum/colorlib/styles.py +++ b/plumbum/colorlib/styles.py @@ -8,6 +8,7 @@ """ +import contextlib import os import platform import re @@ -187,12 +188,10 @@ def from_simple(cls, color, fg=True): return self def _from_simple(self, color): - try: + with contextlib.suppress(AttributeError): color = color.lower() color = color.replace(" ", "") color = color.replace("_", "") - except AttributeError: - pass if color == "reset": return @@ -219,12 +218,10 @@ def from_full(cls, color, fg=True): return self def _from_full(self, color): - try: + with contextlib.suppress(AttributeError): color = color.lower() color = color.replace(" ", "") color = color.replace("_", "") - except AttributeError: - pass if color == "reset": return @@ -567,9 +564,9 @@ def __repr__(self): neg_attributes = ", ".join( f"-{a}" for a in self.attributes if not self.attributes[a] ) - colors = ", ".join(repr(c) for c in [self.fg, self.bg] if c) + colors = ", ".join(repr(c) for c in (self.fg, self.bg) if c) string = ( - "; ".join(s for s in [attributes, neg_attributes, colors] if s) or "empty" + "; ".join(s for s in (attributes, neg_attributes, colors) if s) or "empty" ) if self.isreset: string = "reset" diff --git a/plumbum/commands/base.py b/plumbum/commands/base.py index eafbde37c..7de93da3a 100644 --- a/plumbum/commands/base.py +++ b/plumbum/commands/base.py @@ -1,7 +1,7 @@ +import contextlib import functools import shlex import subprocess -from contextlib import contextmanager from subprocess import PIPE, Popen from tempfile import TemporaryFile from types import MethodType @@ -172,7 +172,7 @@ def nohup(self, cwd=".", stdout="nohup.out", stderr=None, append=True): """Runs a command detached.""" return self.machine.daemonic_popen(self, cwd, stdout, stderr, append) - @contextmanager + @contextlib.contextmanager def bgrun(self, args=(), **kwargs): """Runs the given command as a context manager, allowing you to create a `pipeline `_ (not in the UNIX sense) @@ -215,11 +215,9 @@ def runner(): return run_proc(p, retcode, timeout) finally: del p.run # to break cyclic reference p -> cell -> p - for f in [p.stdin, p.stdout, p.stderr]: - try: + for f in (p.stdin, p.stdout, p.stderr): + with contextlib.suppress(Exception): f.close() - except Exception: - pass p.run = runner yield p diff --git a/plumbum/commands/daemons.py b/plumbum/commands/daemons.py index 82047dd6a..419d9b412 100644 --- a/plumbum/commands/daemons.py +++ b/plumbum/commands/daemons.py @@ -1,3 +1,4 @@ +import contextlib import errno import os import signal @@ -63,10 +64,8 @@ def posix_daemonize(command, cwd, stdout=None, stderr=None, append=True): _, rc = os.waitpid(firstpid, 0) output = os.read(rfd, MAX_SIZE) os.close(rfd) - try: + with contextlib.suppress(UnicodeError): output = output.decode("utf8") - except UnicodeError: - pass if rc == 0 and output.isdigit(): secondpid = int(output) else: diff --git a/plumbum/commands/processes.py b/plumbum/commands/processes.py index 51d930c2f..273398376 100644 --- a/plumbum/commands/processes.py +++ b/plumbum/commands/processes.py @@ -1,4 +1,5 @@ import atexit +import contextlib import heapq import math import time @@ -220,26 +221,22 @@ def _timeout_thread_func(): timeout = max(0, ttk - time.time()) else: timeout = None - try: + with contextlib.suppress(QueueEmpty): proc, time_to_kill = _timeout_queue.get(timeout=timeout) if proc is SystemExit: # terminate return waiting.push((time_to_kill, proc)) - except QueueEmpty: - pass now = time.time() while waiting: ttk, proc = waiting.peek() if ttk > now: break waiting.pop() - try: + with contextlib.suppress(OSError): if proc.poll() is None: proc.kill() proc._timed_out = True - except OSError: - pass except Exception: if _shutting_down: # to prevent all sorts of exceptions during interpreter shutdown diff --git a/plumbum/fs/atomic.py b/plumbum/fs/atomic.py index 1285998e0..520964886 100644 --- a/plumbum/fs/atomic.py +++ b/plumbum/fs/atomic.py @@ -3,9 +3,9 @@ """ import atexit +import contextlib import os import threading -from contextlib import contextmanager from plumbum.machines.local import local @@ -24,7 +24,7 @@ ) raise - @contextmanager + @contextlib.contextmanager def locked_file(fileno, blocking=True): hndl = msvcrt.get_osfhandle(fileno) try: @@ -46,7 +46,7 @@ def locked_file(fileno, blocking=True): else: if hasattr(fcntl, "lockf"): - @contextmanager + @contextlib.contextmanager def locked_file(fileno, blocking=True): fcntl.lockf(fileno, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) try: @@ -56,7 +56,7 @@ def locked_file(fileno, blocking=True): else: - @contextmanager + @contextlib.contextmanager def locked_file(fileno, blocking=True): fcntl.flock(fileno, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) try: @@ -114,7 +114,7 @@ def reopen(self): os.open(str(self.path), os.O_CREAT | os.O_RDWR, 384), "r+b", 0 ) - @contextmanager + @contextlib.contextmanager def locked(self, blocking=True): """ A context manager that locks the file; this function is reentrant by the thread currently @@ -274,10 +274,8 @@ def __exit__(self, t, v, tb): self.release() def __del__(self): - try: + with contextlib.suppress(Exception): self.release() - except Exception: # pylint:disable=broad-except - pass def close(self): self.atomicfile.close() diff --git a/plumbum/machines/local.py b/plumbum/machines/local.py index a0fc36af2..5f1630c90 100644 --- a/plumbum/machines/local.py +++ b/plumbum/machines/local.py @@ -1,3 +1,4 @@ +import contextlib import logging import os import platform @@ -187,15 +188,12 @@ def which(cls, progname): key = (progname, cls.env.get("PATH", "")) - try: + with contextlib.suppress(KeyError): return cls._program_cache[key] - except KeyError: - pass alternatives = [progname] if "_" in progname: - alternatives.append(progname.replace("_", "-")) - alternatives.append(progname.replace("_", ".")) + alternatives += [progname.replace("_", "-"), progname.replace("_", ".")] for pn in alternatives: path = cls._which(pn) if path: diff --git a/plumbum/machines/paramiko_machine.py b/plumbum/machines/paramiko_machine.py index a973d3c2e..6b16d17c3 100644 --- a/plumbum/machines/paramiko_machine.py +++ b/plumbum/machines/paramiko_machine.py @@ -1,3 +1,4 @@ +import contextlib import errno import logging import os @@ -258,11 +259,9 @@ def __init__( ssh_config = paramiko.SSHConfig() with open(os.path.expanduser("~/.ssh/config"), encoding="utf-8") as f: ssh_config.parse(f) - try: + with contextlib.suppress(KeyError): hostConfig = ssh_config.lookup(host) kwargs["sock"] = paramiko.ProxyCommand(hostConfig["proxycommand"]) - except KeyError: - pass self._client.connect(host, **kwargs) self._keep_alive = keep_alive self._sftp = None diff --git a/plumbum/machines/remote.py b/plumbum/machines/remote.py index 9f18421d7..52ab65221 100644 --- a/plumbum/machines/remote.py +++ b/plumbum/machines/remote.py @@ -1,5 +1,5 @@ +import contextlib import re -from contextlib import contextmanager from tempfile import NamedTemporaryFile from plumbum.commands import CommandNotFound, ConcreteCommand, shquote @@ -228,15 +228,12 @@ def which(self, progname): """ key = (progname, self.env.get("PATH", "")) - try: + with contextlib.suppress(KeyError): return self._program_cache[key] - except KeyError: - pass alternatives = [progname] if "_" in progname: - alternatives.append(progname.replace("_", "-")) - alternatives.append(progname.replace("_", ".")) + alternatives += [progname.replace("_", "-"), progname.replace("_", ".")] for name in alternatives: for p in self.env.path: fn = p / name @@ -323,7 +320,7 @@ def pgrep(self, pattern): if pat.search(procinfo.args): yield procinfo - @contextmanager + @contextlib.contextmanager def tempdir(self): """A context manager that creates a remote temporary directory, which is removed when the context exits""" diff --git a/plumbum/machines/session.py b/plumbum/machines/session.py index 3ebba97e6..7ffb042f9 100644 --- a/plumbum/machines/session.py +++ b/plumbum/machines/session.py @@ -1,3 +1,4 @@ +import contextlib import logging import random import threading @@ -241,10 +242,8 @@ def __exit__(self, t, v, tb): self.close() def __del__(self): - try: + with contextlib.suppress(Exception): self.close() - except Exception: - pass def alive(self): """Returns ``True`` if the underlying shell process is alive, ``False`` otherwise""" @@ -254,21 +253,15 @@ def close(self): """Closes (terminates) the shell session""" if not self.alive(): return - try: + with contextlib.suppress(ValueError, OSError): self.proc.stdin.write(b"\nexit\n\n\nexit\n\n") self.proc.stdin.flush() time.sleep(0.05) - except (ValueError, OSError): - pass - for p in [self.proc.stdin, self.proc.stdout, self.proc.stderr]: - try: + for p in (self.proc.stdin, self.proc.stdout, self.proc.stderr): + with contextlib.suppress(Exception): p.close() - except Exception: - pass - try: + with contextlib.suppress(OSError): self.proc.kill() - except OSError: - pass self.proc = None def popen(self, cmd): diff --git a/plumbum/path/base.py b/plumbum/path/base.py index 5f63b35f6..10e1862e2 100644 --- a/plumbum/path/base.py +++ b/plumbum/path/base.py @@ -29,7 +29,7 @@ class Path(str, ABC): CASE_SENSITIVE = True def __repr__(self): - return f"<{self.__class__.__name__} {str(self)}>" + return f"<{self.__class__.__name__} {self}>" def __truediv__(self, other): """Joins two paths""" From da87b67d3efcaa61554756d56775ac44a3379c00 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 5 Oct 2022 16:13:00 -0400 Subject: [PATCH 37/37] feat: add all_markers to Set (#619) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/cli.rst | 8 ++++--- plumbum/cli/switches.py | 53 +++++++++++++++++++++++++++++------------ pyproject.toml | 1 + tests/test_cli.py | 16 +++++++++++-- 4 files changed, 58 insertions(+), 20 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a85af4309..0e48ce521 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -190,9 +190,11 @@ For instance :: ValueError("invalid literal for int() with base 10: 'foo'",) The toolkit includes two additional "types" (or rather, *validators*): ``Range`` and ``Set``. -``Range`` takes a minimal value and a maximal value and expects an integer in that range -(inclusive). ``Set`` takes a set of allowed values, and expects the argument to match one of -these values. Here's an example :: +``Range`` takes a minimal value and a maximal value and expects an integer in +that range (inclusive). ``Set`` takes a set of allowed values, and expects the +argument to match one of these values. You can set ``case_sensitive=False``, or +add ``all_markers={"*", "all"}`` if you want to have a "trigger all markers" +marker. Here's an example :: class MyApp(cli.Application): _port = 8080 diff --git a/plumbum/cli/switches.py b/plumbum/cli/switches.py index 830e0008c..692b599fb 100644 --- a/plumbum/cli/switches.py +++ b/plumbum/cli/switches.py @@ -1,6 +1,8 @@ +import collections.abc import contextlib import inspect from abc import ABC, abstractmethod +from typing import Callable, Generator, List, Union from plumbum import local from plumbum.cli.i18n import get_translation_for @@ -456,41 +458,62 @@ class MyApp(Application): comparison or not. The default is ``False`` :param csv: splits the input as a comma-separated-value before validating and returning a list. Accepts ``True``, ``False``, or a string for the separator + :param all_markers: When a user inputs any value from this set, all values are iterated + over. Something like {"*", "all"} would be a potential setting for + this option. """ - def __init__(self, *values, **kwargs): - self.case_sensitive = kwargs.pop("case_sensitive", False) - self.csv = kwargs.pop("csv", False) - if self.csv is True: - self.csv = "," - if kwargs: - raise TypeError( - _("got unexpected keyword argument(s): {0}").format(kwargs.keys()) - ) + def __init__( + self, + *values: Union[str, Callable[[str], str]], + case_sensitive: bool = False, + csv: Union[bool, str] = False, + all_markers: "collections.abc.Set[str]" = frozenset(), + ) -> None: + self.case_sensitive = case_sensitive + if isinstance(csv, bool): + self.csv = "," if csv else "" + else: + self.csv = csv self.values = values + self.all_markers = all_markers def __repr__(self): items = ", ".join(v if isinstance(v, str) else v.__name__ for v in self.values) return f"{{{items}}}" - def __call__(self, value, check_csv=True): + def _call_iter( + self, value: str, check_csv: bool = True + ) -> Generator[str, None, None]: if self.csv and check_csv: - return [self(v.strip(), check_csv=False) for v in value.split(",")] + for v in value.split(self.csv): + yield from self._call_iter(v.strip(), check_csv=False) + if not self.case_sensitive: value = value.lower() + for opt in self.values: if isinstance(opt, str): if not self.case_sensitive: opt = opt.lower() - if opt == value: - return opt # always return original value + if opt == value or value in self.all_markers: + yield opt # always return original value continue with contextlib.suppress(ValueError): - return opt(value) - raise ValueError(f"Invalid value: {value} (Expected one of {self.values})") + yield opt(value) + + def __call__(self, value: str, check_csv: bool = True) -> Union[str, List[str]]: + items = list(self._call_iter(value, check_csv)) + if not items: + msg = f"Invalid value: {value} (Expected one of {self.values})" + raise ValueError(msg) + if self.csv and check_csv or len(items) > 1: + return items + return items[0] def choices(self, partial=""): choices = {opt if isinstance(opt, str) else f"({opt})" for opt in self.values} + choices |= self.all_markers if partial: choices = {opt for opt in choices if opt.lower().startswith(partial)} return choices diff --git a/pyproject.toml b/pyproject.toml index 27ddb9908..bf9042f4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,4 +105,5 @@ messages_control.disable = [ "too-many-statements", "unidiomatic-typecheck", # TODO: might be able to remove "unnecessary-lambda-assignment", # TODO: 4 instances + "unused-import", # identical to flake8 but has typing false positives ] diff --git a/tests/test_cli.py b/tests/test_cli.py index b7a5e5aa2..c2512a419 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -27,7 +27,9 @@ def bacon(self, param): text wrapping in help messages as well""", ) - csv = cli.SwitchAttr(["--csv"], cli.Set("MIN", "MAX", int, csv=True)) + csv = cli.SwitchAttr( + ["--csv"], cli.Set("MIN", "MAX", int, csv=True, all_markers={"all"}) + ) num = cli.SwitchAttr(["--num"], cli.Set("MIN", "MAX", int)) def main(self, *args): @@ -36,6 +38,8 @@ def main(self, *args): self.eggs = old self.tailargs = args + print(self.csv) + class PositionalApp(cli.Application): def main(self, one): @@ -163,7 +167,7 @@ def test_meta_switches(self): _, rc = SimpleApp.run(["foo", "--version"], exit=False) assert rc == 0 - def test_okay(self): + def test_okay(self, capsys): _, rc = SimpleApp.run(["foo", "--bacon=81"], exit=False) assert rc == 0 @@ -195,6 +199,14 @@ def test_okay(self): _, rc = SimpleApp.run(["foo", "--bacon=81", "--num=100"], exit=False) assert rc == 0 + capsys.readouterr() + _, rc = SimpleApp.run(["foo", "--bacon=81", "--csv=all,100"], exit=False) + assert rc == 0 + output = capsys.readouterr() + assert "min" in output.out + assert "max" in output.out + assert "100" in output.out + _, rc = SimpleApp.run(["foo", "--bacon=81", "--num=MAX"], exit=False) assert rc == 0