From 45f6458cc5ccd7cf96d77bde800968fd4b6fa83a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 8 Feb 2022 11:24:17 +0000 Subject: [PATCH 1/3] aio.core: Add interactive utils Signed-off-by: Ryan Northey --- aio.core/aio/core/BUILD | 4 + aio.core/aio/core/__init__.py | 2 + aio.core/aio/core/interactive/__init__.py | 14 ++ .../aio/core/interactive/abstract/__init__.py | 7 + .../core/interactive/abstract/interactive.py | 159 ++++++++++++++++++ aio.core/aio/core/interactive/interactive.py | 26 +++ aio.core/aio/core/output/__init__.py | 6 +- aio.core/aio/core/output/abstract/__init__.py | 3 +- aio.core/aio/core/output/abstract/output.py | 41 +++++ aio.core/aio/core/output/output.py | 5 + 10 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 aio.core/aio/core/interactive/__init__.py create mode 100644 aio.core/aio/core/interactive/abstract/__init__.py create mode 100644 aio.core/aio/core/interactive/abstract/interactive.py create mode 100644 aio.core/aio/core/interactive/interactive.py diff --git a/aio.core/aio/core/BUILD b/aio.core/aio/core/BUILD index 469a0d8f0..26cd65161 100644 --- a/aio.core/aio/core/BUILD +++ b/aio.core/aio/core/BUILD @@ -34,6 +34,10 @@ pytooling_library( "functional/utils.py", "log/__init__.py", "log/logging.py", + "interactive/abstract/__init__.py", + "interactive/abstract/interactive.py", + "interactive/exceptions.py", + "interactive/interactive.py", "output/abstract/__init__.py", "output/abstract/output.py", "output/exceptions.py", diff --git a/aio.core/aio/core/__init__.py b/aio.core/aio/core/__init__.py index 02c3973ee..d90cd04a8 100644 --- a/aio.core/aio/core/__init__.py +++ b/aio.core/aio/core/__init__.py @@ -4,6 +4,7 @@ directory, event, functional, + interactive, output, stream, subprocess, @@ -15,6 +16,7 @@ "directory", "event", "functional", + "interactive", "output", "stream", "subprocess", diff --git a/aio.core/aio/core/interactive/__init__.py b/aio.core/aio/core/interactive/__init__.py new file mode 100644 index 000000000..122939866 --- /dev/null +++ b/aio.core/aio/core/interactive/__init__.py @@ -0,0 +1,14 @@ + + +from .abstract import AInteractive, APrompt +from .interactive import Interactive, interactive, Prompt +from . import exceptions + + +__all__ = ( + "AInteractive", + "APrompt", + "exceptions", + "interactive", + "Interactive", + "Prompt") diff --git a/aio.core/aio/core/interactive/abstract/__init__.py b/aio.core/aio/core/interactive/abstract/__init__.py new file mode 100644 index 000000000..fd0040883 --- /dev/null +++ b/aio.core/aio/core/interactive/abstract/__init__.py @@ -0,0 +1,7 @@ + +from .interactive import AInteractive, APrompt + + +__all__ = ( + "AInteractive", + "APrompt") diff --git a/aio.core/aio/core/interactive/abstract/interactive.py b/aio.core/aio/core/interactive/abstract/interactive.py new file mode 100644 index 000000000..2a3803a2b --- /dev/null +++ b/aio.core/aio/core/interactive/abstract/interactive.py @@ -0,0 +1,159 @@ + +import asyncio +import re +import sys +import time +from functools import cached_property, partial +from typing import Union + +import abstracts + +from aio.core import functional, output, subprocess +from aio.core.functional import async_property, AwaitableGenerator + + +class APrompt(metaclass=abstracts.Abstraction): + + def __init__(self, match, match_type="any"): + self._match = match + self.match_type = match + + @cached_property + def re_match(self): + return re.compile(self._match) + + def matches(self, counter, output): + # print(counter) + if isinstance(self._match, int): + if counter.get("stdout", 0) >= self._match: + return True + return bool(self.re_match.match(str(output))) + + +class AInteractive(metaclass=abstracts.Abstraction): + + def __init__(self, cmd, prompt, flush_delay=0, wait_for_prompt=True, start_prompt=None): + self.cmd = cmd + self._prompt = prompt + self._start_prompt = start_prompt or prompt + self.flush_delay = flush_delay + self.wait_for_prompt = wait_for_prompt + + @cached_property + def buffer(self): + return asyncio.Queue() + + @cached_property + def prompt(self): + return ( + self.prompt_class(self._prompt) + if not isinstance(self._prompt, self.prompt_class) + else prompt) + + @cached_property + def start_prompt(self): + return ( + self.prompt_class(self._start_prompt) + if not isinstance(self._start_prompt, self.prompt_class) + else start_prompt) + + @property + def prompt_class(self): + return Prompt + + @cached_property + def write_lock(self): + return asyncio.Lock() + + @cached_property + def q(self): + return asyncio.Queue() + + @async_property(cache=True) + async def proc(self): + return await asyncio.create_subprocess_shell( + self.cmd, + # shell=True, + # universal_newlines=True, + stdin=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE) + + async def connect_outputs(self): + await self.stdout_listener + await self.stderr_listener + + @async_property(cache=True) + async def stderr_listener(self): + return asyncio.create_task( + self.listen_to_pipe( + "stderr", + (await self.proc).stderr)) + + @async_property(cache=True) + async def stdout_listener(self): + return asyncio.create_task( + self.listen_to_pipe( + "stdout", + (await self.proc).stdout)) + + async def interact(self, message=None): + await self.send_stdin(message) + counter = dict() + returns = False + while True: + result = await self.q.get() + yield result + counter[result.type] = counter.get(result.type, 0) + 1 + await self.buffer.get() + self.buffer.task_done + if self.interaction_returns(counter, result): + returns = True + if returns and await self.finished_reading: + break + + @async_property + async def finished_reading(self): + if self.buffer.qsize(): + return False + if not self.flush_delay: + return True + await asyncio.sleep(self.flush_delay) + return not self.buffer.qsize() + + def interaction_returns(self, counter, result): + return self.prompt.matches(counter, result) + + async def send_stdin(self, message): + print(f"SEND STDIN {message}") + async with self.write_lock: + proc = await self.proc + if message is not None: + proc.stdin.write(message) + await proc.stdin.drain() + + async def listen_to_pipe(self, type, pipe): + while True: + result = await pipe.readline() + await self.buffer.put(None) + # If we havent completed writing, wait + async with self.write_lock: + # print(f"GOT RESULT: {type} {result}") + await self.q.put(output.CapturedOutput(type, result)) + + async def start(self): + await self.connect_outputs() + print("\n".join(str(h) for h in await self.header)) + self._started = True + + _started = False + + @cached_property + def header(self): + return ( + self(b"") + if self.wait_for_prompt + else None) + + def __call__(self, message=None): + return AwaitableGenerator(self.interact(message)) diff --git a/aio.core/aio/core/interactive/interactive.py b/aio.core/aio/core/interactive/interactive.py new file mode 100644 index 000000000..fdc03d690 --- /dev/null +++ b/aio.core/aio/core/interactive/interactive.py @@ -0,0 +1,26 @@ + +import contextlib + +import abstracts + +from aio.core import interactive + + +@abstracts.implementer(interactive.APrompt) +class Prompt: + pass + + +@abstracts.implementer(interactive.AInteractive) +class Interactive: + + @property + def prompt_class(self): + return Prompt + + +@contextlib.asynccontextmanager +async def interactive(*args, **kwargs): + interaction = Interactive(*args, **kwargs) + await interaction.start() + yield interaction diff --git a/aio.core/aio/core/output/__init__.py b/aio.core/aio/core/output/__init__.py index 5f556efdd..ac947e33c 100644 --- a/aio.core/aio/core/output/__init__.py +++ b/aio.core/aio/core/output/__init__.py @@ -1,15 +1,17 @@ -from .abstract import ACapturedOutput, ABufferedOutputs, AQueueIO -from .output import BufferedOutputs, CapturedOutput, QueueIO +from .abstract import ACapturedOutput, ACapturedOutputs, ABufferedOutputs, AQueueIO +from .output import BufferedOutputs, CapturedOutput, CapturedOutputs, QueueIO from . import exceptions __all__ = ( "ACapturedOutput", + "ACapturedOutputs", "ABufferedOutputs", "AQueueIO", "BufferedOutputs", "CapturedOutput", + "CapturedOutputs", "exceptions", "output", "QueueIO") diff --git a/aio.core/aio/core/output/abstract/__init__.py b/aio.core/aio/core/output/abstract/__init__.py index db9695085..267e149b7 100644 --- a/aio.core/aio/core/output/abstract/__init__.py +++ b/aio.core/aio/core/output/abstract/__init__.py @@ -1,9 +1,10 @@ -from .output import ACapturedOutput, AQueueIO, ABufferedOutputs +from .output import ACapturedOutput, ACapturedOutputs, AQueueIO, ABufferedOutputs __all__ = ( "ACapturedOutput", + "ACapturedOutputs", "AQueueIO", "ABufferedOutputs") diff --git a/aio.core/aio/core/output/abstract/output.py b/aio.core/aio/core/output/abstract/output.py index 0c53a7bc7..d52a6e617 100644 --- a/aio.core/aio/core/output/abstract/output.py +++ b/aio.core/aio/core/output/abstract/output.py @@ -11,11 +11,52 @@ import abstracts from aio import core +from aio.core import functional DEATH_SENTINEL = object() +class ACapturedOutputs(metaclass=abstracts.Abstraction): + """Wraps a list of captured outputs and allows you to + print them, or filter them base on type.""" + + def __init__(self, outputs, output_types=None, out_file=None): + self._outputs = outputs + self._output_types = output_types + self.out_file = functional.maybe_coro(out_file or print) + + def __getitem__(self, type): + return list(self.output_for(type)) + + @property + def output(self): + return "\n".join( + f"{result.type}: {str(result)}" + for result in self._outputs) + + @cached_property + def output_types(self): + if self._output_types: + return self._output_types + return dict( + stdout=sys.stdout, + stderr=sys.stderr) + + async def drain(self, type=None): + types = [type] if type else self.output_types.keys() + for output_type in types: + for output in self[output_type]: + await self.out_file( + output, + file=self.output_types[output_type]) + + def output_for(self, type): + for result in self._outputs: + if result.type == type: + yield result + + class ACapturedOutput(metaclass=abstracts.Abstraction): """Captured output of a given type, eg `stdout`, `stderr`""" diff --git a/aio.core/aio/core/output/output.py b/aio.core/aio/core/output/output.py index 9af9dedea..eab2d1fbf 100644 --- a/aio.core/aio/core/output/output.py +++ b/aio.core/aio/core/output/output.py @@ -29,6 +29,11 @@ class CapturedOutput: pass +@abstracts.implementer(output.ACapturedOutputs) +class CapturedOutputs: + pass + + @abstracts.implementer(output.AQueueIO) class QueueIO: pass From b85f1c081c550848b1458f83d162de632cb884e9 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 8 Feb 2022 17:21:55 +0000 Subject: [PATCH 2/3] libs: Add `aio.api.aspell` --- aio.api.aspell/BUILD | 2 + aio.api.aspell/README.rst | 5 + aio.api.aspell/VERSION | 1 + aio.api.aspell/aio/api/aspell/BUILD | 16 ++ aio.api.aspell/aio/api/aspell/__init__.py | 16 ++ .../aio/api/aspell/abstract/__init__.py | 6 + aio.api.aspell/aio/api/aspell/abstract/api.py | 139 ++++++++++++++++++ aio.api.aspell/aio/api/aspell/api.py | 13 ++ aio.api.aspell/aio/api/aspell/exceptions.py | 0 aio.api.aspell/aio/api/aspell/py.typed | 0 aio.api.aspell/aio/api/aspell/utils.py | 14 ++ aio.api.aspell/setup.cfg | 48 ++++++ aio.api.aspell/setup.py | 5 + aio.api.aspell/tests/BUILD | 12 ++ 14 files changed, 277 insertions(+) create mode 100644 aio.api.aspell/BUILD create mode 100644 aio.api.aspell/README.rst create mode 100644 aio.api.aspell/VERSION create mode 100644 aio.api.aspell/aio/api/aspell/BUILD create mode 100644 aio.api.aspell/aio/api/aspell/__init__.py create mode 100644 aio.api.aspell/aio/api/aspell/abstract/__init__.py create mode 100644 aio.api.aspell/aio/api/aspell/abstract/api.py create mode 100644 aio.api.aspell/aio/api/aspell/api.py create mode 100644 aio.api.aspell/aio/api/aspell/exceptions.py create mode 100644 aio.api.aspell/aio/api/aspell/py.typed create mode 100644 aio.api.aspell/aio/api/aspell/utils.py create mode 100644 aio.api.aspell/setup.cfg create mode 100644 aio.api.aspell/setup.py create mode 100644 aio.api.aspell/tests/BUILD diff --git a/aio.api.aspell/BUILD b/aio.api.aspell/BUILD new file mode 100644 index 000000000..6bd56f3a2 --- /dev/null +++ b/aio.api.aspell/BUILD @@ -0,0 +1,2 @@ + +pytooling_package("aio.api.aspell") diff --git a/aio.api.aspell/README.rst b/aio.api.aspell/README.rst new file mode 100644 index 000000000..55d130147 --- /dev/null +++ b/aio.api.aspell/README.rst @@ -0,0 +1,5 @@ + +aio.api.aspell +============== + +Wrapper around aspell diff --git a/aio.api.aspell/VERSION b/aio.api.aspell/VERSION new file mode 100644 index 000000000..ccf3e968d --- /dev/null +++ b/aio.api.aspell/VERSION @@ -0,0 +1 @@ +0.0.5-dev diff --git a/aio.api.aspell/aio/api/aspell/BUILD b/aio.api.aspell/aio/api/aspell/BUILD new file mode 100644 index 000000000..a73185946 --- /dev/null +++ b/aio.api.aspell/aio/api/aspell/BUILD @@ -0,0 +1,16 @@ + +pytooling_library( + "aio.api.aspell", + dependencies=[ + "//deps:abstracts", + "//deps:aio.core", + ], + sources=[ + "abstract/__init__.py", + "abstract/api.py", + "__init__.py", + "api.py", + "exceptions.py", + "utils.py", + ], +) diff --git a/aio.api.aspell/aio/api/aspell/__init__.py b/aio.api.aspell/aio/api/aspell/__init__.py new file mode 100644 index 000000000..09ad1f03e --- /dev/null +++ b/aio.api.aspell/aio/api/aspell/__init__.py @@ -0,0 +1,16 @@ +"""aio.api.aspell.""" + +from . import abstract +from . import exceptions +from . import utils +from .api import ( + AspellAPI, ) +from .abstract import ( + AAspellAPI, ) + + +__all__ = ( + "abstract", + "AAspellAPI", + "AspellAPI", + "exceptions", ) diff --git a/aio.api.aspell/aio/api/aspell/abstract/__init__.py b/aio.api.aspell/aio/api/aspell/abstract/__init__.py new file mode 100644 index 000000000..0acfe6318 --- /dev/null +++ b/aio.api.aspell/aio/api/aspell/abstract/__init__.py @@ -0,0 +1,6 @@ + +from .api import AAspellAPI + + +__all__ = ( + "AAspellAPI", ) diff --git a/aio.api.aspell/aio/api/aspell/abstract/api.py b/aio.api.aspell/aio/api/aspell/abstract/api.py new file mode 100644 index 000000000..e99b7bf54 --- /dev/null +++ b/aio.api.aspell/aio/api/aspell/abstract/api.py @@ -0,0 +1,139 @@ + +import asyncio +import abc +import shutil +from functools import cached_property +from typing import Any, Type + +import abstracts + +from aio.core.functional import async_property +from aio.core.tasks import concurrent +from aio.core.interactive import Interactive + + +class AspellPipe: + handlers = [] + + def __init__(self, in_q, out_q): + self.in_q = in_q + self.out_q = out_q + + @async_property(cache=True) + async def proc(self): + return await asyncio.create_subprocess_exec( + "/home/phlax/.virtualenvs/envoydev/pytooling/echo.py", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + + async def listen(self): + print ("STARTING LISTENER") + while True: + msg = await self.in_q.get() + print(f"IN Q MESSAGE RECVD: {msg}") + self.in_q.task_done() + print(f"Sending message to process: {msg}") + result = await self.write(msg) + await self.out_q.put(result) + + async def start(self): + print ("STARTING ASPELL") + self.task = asyncio.create_task(self.listen()) + return (await self.write(b""))[0] + + async def stop(self): + print ("STOPPING ASPELL") + + async def write(self, message): + print(f"WRITE: {message}") + proc = await self.proc + stdout, stderr = await proc.communicate(message) + other = await proc.stdout.read() + more = await proc.communicate(message) + return stdout, stderr + + +class MultiPipe: + + def __init__(self, pipe_type): + self.pipe_type = pipe_type + + @cached_property + def in_q(self): + return asyncio.Queue() + + @cached_property + def out_q(self): + return asyncio.Queue() + + @async_property(cache=True) + async def pipes(self): + aspell_pipe = self.pipe_type(self.in_q, self.out_q) + print(f"Started aspell pipe: {await aspell_pipe.start()}") + return aspell_pipe + + async def write(self, message): + print(f"PUTTING MESSAGE: {message}") + print(f"IN Q: {self.in_q}") + await self.in_q.put(message) + print(f"DONE") + while True: + stdout, stderr = await self.out_q.get() + self.out_q.task_done() + if stdout or stderr: + return stdout, stderr + + +class AAspellAPI(metaclass=abstracts.Abstraction): + """Aspell API wrapper. + """ + + def __init__(self, *args, **kwargs) -> None: + self.args = args + self.kwargs = kwargs + + @cached_property + def aspell_command(self): + command = shutil.which("aspell") + return f"{command} -a" + + @async_property(cache=True) + async def session(self): + session = Interactive(self.aspell_command, 1) + await session.start() + return session + + async def compile_dictionary(self, path): + # breakpoint() + pass + + @async_property(cache=True) + async def pipe(self): + aspell_pipe = MultiPipe(AspellPipe) + await aspell_pipe.pipes + # await aspell_pipe.start() + return aspell_pipe + + async def listener(self): + print(f"MESSAGE RCVD: {stdout} {stderr}") + + async def spellcheck(self, message): + pipe = await self.pipe + return await pipe.write(message) + + async def start(self): + await self.session + + async def compile_dictionary(self, dictionary): + words = ["asdfasfdafds", "cabbage"] + session = await self.session + for word in words: + response = await session(f"{word}\n".encode("utf-8")) + if str(response[0]).strip() == "*": + print(f"{word} is a good word") + else: + print(f"{word} is a bad word") + + async def stop(self): + pass diff --git a/aio.api.aspell/aio/api/aspell/api.py b/aio.api.aspell/aio/api/aspell/api.py new file mode 100644 index 000000000..e28fce06c --- /dev/null +++ b/aio.api.aspell/aio/api/aspell/api.py @@ -0,0 +1,13 @@ + +from typing import Type + +import abstracts + +from .abstract import AAspellAPI + + +@abstracts.implementer(AAspellAPI) +class AspellAPI: + + def __init__(self, *args, **kwargs) -> None: + AAspellAPI.__init__(self, *args, **kwargs) diff --git a/aio.api.aspell/aio/api/aspell/exceptions.py b/aio.api.aspell/aio/api/aspell/exceptions.py new file mode 100644 index 000000000..e69de29bb diff --git a/aio.api.aspell/aio/api/aspell/py.typed b/aio.api.aspell/aio/api/aspell/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/aio.api.aspell/aio/api/aspell/utils.py b/aio.api.aspell/aio/api/aspell/utils.py new file mode 100644 index 000000000..bbac2fab3 --- /dev/null +++ b/aio.api.aspell/aio/api/aspell/utils.py @@ -0,0 +1,14 @@ + +from datetime import datetime + + +# these only deal with utc but are good enough for working with the +# github api + + +def dt_from_js_isoformat(iso: str) -> datetime: + return datetime.fromisoformat(iso.replace("Z", "+00:00")) + + +def dt_to_js_isoformat(dt: datetime) -> str: + return dt.isoformat().replace("+00:00", "Z") diff --git a/aio.api.aspell/setup.cfg b/aio.api.aspell/setup.cfg new file mode 100644 index 000000000..9edfddc27 --- /dev/null +++ b/aio.api.aspell/setup.cfg @@ -0,0 +1,48 @@ +[metadata] +name = aio.api.aspell +version = file: VERSION +author = Ryan Northey +author_email = ryan@synca.io +maintainer = Ryan Northey +maintainer_email = ryan@synca.io +license = Apache Software License 2.0 +url = https://github.com/envoyproxy/pytooling/tree/main/aio.api.aspell +description = Wrapper around aspell +long_description = file: README.rst +classifiers = + Development Status :: 4 - Beta + Framework :: Pytest + Intended Audience :: Developers + Topic :: Software Development :: Testing + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: Implementation :: CPython + Operating System :: OS Independent + License :: OSI Approved :: Apache Software License + +[options] +python_requires = >=3.5 +py_modules = aio.api.aspell +packages = find_namespace: +install_requires = + abstracts>=0.0.12 + aio.core>=0.3.0 + gidgethub + packaging + +[options.extras_require] +test = + pytest + pytest-asyncio + pytest-coverage + pytest-patches +lint = flake8 +types = + mypy +publish = wheel + +[options.package_data] +* = py.typed diff --git a/aio.api.aspell/setup.py b/aio.api.aspell/setup.py new file mode 100644 index 000000000..1f6a64b9c --- /dev/null +++ b/aio.api.aspell/setup.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from setuptools import setup # type:ignore + +setup() diff --git a/aio.api.aspell/tests/BUILD b/aio.api.aspell/tests/BUILD new file mode 100644 index 000000000..935a7c463 --- /dev/null +++ b/aio.api.aspell/tests/BUILD @@ -0,0 +1,12 @@ + +pytooling_tests( + "aio.api.github", + dependencies=[ + "//deps:abstracts", + "//deps:aio.core", + "//deps:aiohttp", + "//deps:gidgethub", + "//deps:packaging", + "//deps:pytest-asyncio", + ], +) From f3b3e22218f00a3108d39e7e481525586e4fb279 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 7 Feb 2022 04:41:12 +0000 Subject: [PATCH 3/3] `envoy.code.check`: v0.0.2 Signed-off-by: Ryan Northey --- envoy.code.check/tests/test_checker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/envoy.code.check/tests/test_checker.py b/envoy.code.check/tests/test_checker.py index 0bb9d208e..6c937d03c 100644 --- a/envoy.code.check/tests/test_checker.py +++ b/envoy.code.check/tests/test_checker.py @@ -38,6 +38,8 @@ def test_checker_constructor(patches, args, kwargs): assert "glint_class" not in directory.__dict__ assert checker.shellcheck_class == check.ShellcheckCheck assert "shellcheck_class" not in directory.__dict__ + assert checker.glint_class == check.GlintCheck + assert "glint_class" not in directory.__dict__ assert checker.yapf_class == check.YapfCheck assert "yapf_class" not in directory.__dict__ @@ -66,6 +68,7 @@ def test_checker_path(patches): [check.Flake8Check, check.GlintCheck, check.ShellcheckCheck, + check.GlintCheck, check.YapfCheck]) def test_checker_constructors(patches, args, kwargs, sub): patched = patches(