From 7bf87ba3bf1bf75325c9951fc55a49e8f7a1fa89 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 30 Oct 2022 23:35:38 -0400 Subject: [PATCH 01/42] WIP: Enable import with format [:path.to.object] --- .vscode/launch.json | 12 ++ .vscode/settings.json | 3 +- lint.sh | 3 - pyproject.toml | 1 + src/aiotaskq/__init__.py | 6 + src/aiotaskq/app.py | 108 ++++++++++++++++++ src/aiotaskq/exceptions.py | 4 + src/aiotaskq/task.py | 2 + src/aiotaskq/tests/__init__.py | 3 + src/aiotaskq/tests/apps/__init__.py | 5 + .../tests/apps/simple_app_celery/__init__.py | 0 .../tests/apps/simple_app_celery/app.py | 8 ++ .../tests/apps/simple_app_celery/celery.py | 11 ++ .../tests/apps/simple_app_celery/tasks.py | 32 ++++++ .../apps/simple_app_encapsulated/__init__.py | 6 + .../apps/simple_app_encapsulated/aiotaskq.py | 12 ++ .../tests/apps/simple_app_encapsulated/app.py | 8 ++ .../some_module/__init__.py | 0 .../some_module/tasks.py | 6 + .../some_other_module/__init__.py | 0 .../some_other_module/some_tasks.py | 0 .../simple_app_encapsulated_2/__init__.py | 6 + .../apps/simple_app_encapsulated_2/app.py | 8 ++ .../some_file_name.py | 14 +++ .../some_module/__init__.py | 0 .../some_module/tasks.py | 6 + .../some_other_module/__init__.py | 0 .../some_other_module/some_tasks.py | 0 src/aiotaskq/tests/test_app.py | 60 ++++++++++ src/aiotaskq/worker.py | 8 +- test.sh | 5 +- 31 files changed, 325 insertions(+), 12 deletions(-) create mode 100644 src/aiotaskq/app.py create mode 100644 src/aiotaskq/tests/__init__.py create mode 100644 src/aiotaskq/tests/apps/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_celery/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_celery/app.py create mode 100644 src/aiotaskq/tests/apps/simple_app_celery/celery.py create mode 100644 src/aiotaskq/tests/apps/simple_app_celery/tasks.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/app.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/some_tasks.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/__init__.py create mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/some_tasks.py create mode 100644 src/aiotaskq/tests/test_app.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 0b6c3e1..0b568c3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -43,6 +43,18 @@ "tests.apps.simple_app" ], "console": "integratedTerminal", + }, + { + "name": "Sample Worker (Simple App Celery)", + "type": "python", + "request": "launch", + "module": "celery", + "args": [ + "-A", + "aiotaskq.tests.apps.simple_app_celery", + "worker", + ], + "justMyCode": false } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index ebf58cd..9390569 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,5 +12,6 @@ "python.testing.unittestEnabled": false, "python.analysis.extraPaths": [ "src/tests/" - ] + ], + "python.defaultInterpreterPath": "../.venv/bin/python3" } diff --git a/lint.sh b/lint.sh index 6041be0..f135f39 100755 --- a/lint.sh +++ b/lint.sh @@ -1,6 +1,3 @@ -pip install --upgrade pip -pip install -e .[dev] - if [ -z $1 ]; then pylint -v --rcfile ./.pylintrc src/aiotaskq diff --git a/pyproject.toml b/pyproject.toml index ed11e53..9fe11e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ license = { file = "LICENSE" } [project.optional-dependencies] dev = [ "black >= 22.1.0, < 22.2.0", + "celery >= 5.2.0 < 5.3.0", "coverage >= 6.4.0, < 6.5.0", "mypy >= 0.931, < 1.0", "mypy-extensions >= 0.4.0, < 0.5.0", diff --git a/src/aiotaskq/__init__.py b/src/aiotaskq/__init__.py index b38e566..4709471 100644 --- a/src/aiotaskq/__init__.py +++ b/src/aiotaskq/__init__.py @@ -27,7 +27,9 @@ async def main(): """ +import os from pathlib import Path +from typing import TYPE_CHECKING import tomlkit @@ -38,3 +40,7 @@ async def main(): __version__ = toml_dict["project"]["version"] __all__ = ["__version__", "task"] + +# if os.environ["ENV"] == "test" or TYPE_CHECKING: +from . import tests +__all__ += ["tests"] diff --git a/src/aiotaskq/app.py b/src/aiotaskq/app.py new file mode 100644 index 0000000..63c8c54 --- /dev/null +++ b/src/aiotaskq/app.py @@ -0,0 +1,108 @@ +import importlib +import inspect +import typing as t + +from .exceptions import AppImportError +from .task import task, Task, P, RT + + +class Aiotaskq: + """Encapsulate the whole application logic.""" + + import_path: str + task_map: dict[str, Task] = {} + + def __init__(self, import_path: t.Optional[str] = None, tasks: t.Optional[list[Task]] = None): + self.import_path = ( + import_path + if import_path is not None + else inspect.getmodule(self).__name__ # type: ignore + ) + self.task_map = {task_.__name__: task_ for task_ in tasks} if tasks else {} + + def __getattribute__(self, attr_name: str) -> t.Any: + try: + return object.__getattribute__(self, attr_name) + except AttributeError: + # The desired attribute could be a Task. + task_map: dict[str, Task] = object.__getattribute__(self, "task_map") + try: + return task_map[attr_name] + except KeyError as exc: + raise AttributeError from exc + + @classmethod + def from_import_path(cls, app_or_module_path: str) -> "Aiotaskq": + """Return an Aiotaskq instance instantied from the provided app_path string.""" + + app_import_error_msg = f"App or module path `{app_or_module_path}` is not valid." + + app: "Aiotaskq" + if ":" in app_or_module_path: + print("\n0") + # Import path points to an Aiotaskq instance -- use it. + module_path, app_name = app_or_module_path.split(":") + + try: + module = importlib.import_module(module_path) + except ImportError as exc: + raise AppImportError(app_import_error_msg) from exc + + app = getattr(module, app_name) + else: + try: + app_or_module = importlib.import_module(f"{app_or_module_path}.aiotaskq") + except ImportError: + try: + app_or_module = importlib.import_module(app_or_module_path) + except ImportError as exc: + raise AppImportError(app_import_error_msg) from exc + + if isinstance(app_or_module, Aiotaskq): + # Import path points to an Aiotaskq instance -- use it. + print("\n1") + app = app_or_module + elif hasattr(app_or_module, "app") and isinstance(getattr(app_or_module, "app"), Aiotaskq): + # Import path points to a module that contains an Aiotaskq instance + # named as `app` -- retrieve the instance and use it. + print("\n2") + app = getattr(app_or_module, "app") + elif ( + hasattr(app_or_module, "aiotaskq") + and isinstance(getattr(app_or_module, "aiotaskq"), Aiotaskq) + ): + # Import path points to a module that contains an Aiotaskq instance + # named as `aiotaskq` -- retrieve the instance and use it. + print("\n3") + app = getattr(app_or_module, "aiotaskq") + else: + # Import path points to neither an Aiotaskq instance, nor a module + # containing an Aiotaskq instance. + tasks = cls._extract_tasks(app_or_module=app_or_module) + if not tasks: + # The Aiotaskq instance or module does not contain any + # Task -- Invalid import path. + raise AppImportError(app_import_error_msg) + # The Aiotaskq instance or module contains some Task -- instantiate a + # new Aiotaskq instance and return it. + print(f"\n4: {dict((t.__name__, t) for t in tasks)}") + app = cls(tasks=tasks) + + app.import_path = app_or_module_path + return app + + def register_task(self, func: t.Callable[P, RT]) -> Task[P, RT]: + """ + Register a function as a task so that it can be retrieved from an Aiotaskq app instance. + """ + task_: Task[P, RT] = task(func) + self.task_map[func.__name__] = task_ + return task_ + + @classmethod + def _extract_tasks(cls, app_or_module: t.Any) -> list[Task]: + return [ + getattr(app_or_module, attr) + for attr in app_or_module.__dict__ + if isinstance(getattr(app_or_module, attr), Task) + ] diff --git a/src/aiotaskq/exceptions.py b/src/aiotaskq/exceptions.py index 4f58a4f..8ee82b1 100644 --- a/src/aiotaskq/exceptions.py +++ b/src/aiotaskq/exceptions.py @@ -15,3 +15,7 @@ class UrlNotSupported(Exception): class ConcurrencyTypeNotSupported(Exception): """This concurrency type is currently not supported.""" + + +class AppImportError(ImportError): + """App or module import path provided is invalid.""" diff --git a/src/aiotaskq/task.py b/src/aiotaskq/task.py index 369e2f4..b084115 100644 --- a/src/aiotaskq/task.py +++ b/src/aiotaskq/task.py @@ -73,6 +73,7 @@ def some_func(x: int, y: int) -> int: ``` """ + __name__: str __qualname__: str def __init__(self, func: t.Callable[P, RT]) -> None: @@ -142,6 +143,7 @@ def task(func: t.Callable[P, RT]) -> Task[P, RT]: ] ) task_ = Task[P, RT](func) + task_.__name__ = func.__name__ task_.__qualname__ = f"{module_path}.{func.__name__}" task_.__module__ = module_path return task_ diff --git a/src/aiotaskq/tests/__init__.py b/src/aiotaskq/tests/__init__.py new file mode 100644 index 0000000..4c1e2f0 --- /dev/null +++ b/src/aiotaskq/tests/__init__.py @@ -0,0 +1,3 @@ +from . import apps + +__all__ = ["apps"] diff --git a/src/aiotaskq/tests/apps/__init__.py b/src/aiotaskq/tests/apps/__init__.py new file mode 100644 index 0000000..7337e91 --- /dev/null +++ b/src/aiotaskq/tests/apps/__init__.py @@ -0,0 +1,5 @@ +from . import simple_app +from . import simple_app_encapsulated +from . import simple_app_encapsulated_2 + +__all__ = ["simple_app", "simple_app_encapsulated", "simple_app_encapsulated_2"] diff --git a/src/aiotaskq/tests/apps/simple_app_celery/__init__.py b/src/aiotaskq/tests/apps/simple_app_celery/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/aiotaskq/tests/apps/simple_app_celery/app.py b/src/aiotaskq/tests/apps/simple_app_celery/app.py new file mode 100644 index 0000000..1b74d1c --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_celery/app.py @@ -0,0 +1,8 @@ +from aiotaskq.tests.apps.simple_app_celery.tasks import get_formula +from .celery import app + + +if __name__ == "__main__": + formula = get_formula() + ret = formula.apply_async().get() + assert ret == 14 diff --git a/src/aiotaskq/tests/apps/simple_app_celery/celery.py b/src/aiotaskq/tests/apps/simple_app_celery/celery.py new file mode 100644 index 0000000..fd4311b --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_celery/celery.py @@ -0,0 +1,11 @@ +from celery import Celery + + +app = Celery( + include=["aiotaskq.tests.apps.simple_app_celery.tasks"], +) +app.conf.broker_url = 'redis://localhost:6379/0' +app.conf.result_backend = 'redis://localhost:6379/0' + +if __name__ == "__main__": + app.start() diff --git a/src/aiotaskq/tests/apps/simple_app_celery/tasks.py b/src/aiotaskq/tests/apps/simple_app_celery/tasks.py new file mode 100644 index 0000000..11f4699 --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_celery/tasks.py @@ -0,0 +1,32 @@ +import time +from celery import chord, group +import celery + +from .celery import app + + +@app.task() +def add(ls: list[int]) -> int: + time.sleep(5) + return sum(x for x in ls) + + +@app.task() +def times(x: int, y: int) -> int: + time.sleep(2) + return x * y + + +def get_formula(): + """ + Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). + """ + + times_tasks: list[celery.Task] = [ + times.si(x=1, y=2), + times.si(x=3, y=4), + ] + return chord( + header=group(times_tasks), + body=add.s(), + ) diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py new file mode 100644 index 0000000..abdcab5 --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py @@ -0,0 +1,6 @@ +from . import aiotaskq +from . import app +from . import some_module +from . import some_other_module + +__all__ = ["aiotaskq", "app", "some_module", "some_other_module"] diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py new file mode 100644 index 0000000..500d6e0 --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py @@ -0,0 +1,12 @@ +import time + +from aiotaskq.app import Aiotaskq + +app = Aiotaskq( + # includes=[".tasks"], +) + +@app.register_task +def add(ls: list[int]) -> int: + time.sleep(5) + return sum(x for x in ls) diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/app.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/app.py new file mode 100644 index 0000000..905a39f --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated/app.py @@ -0,0 +1,8 @@ +"""A sample module that contains an app instance that encapsulates all app logic.""" + + +class SomeSimpleApp: + pass + + +some_simple_app = SomeSimpleApp() diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py new file mode 100644 index 0000000..6557315 --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py @@ -0,0 +1,6 @@ +from ..aiotaskq import app + + +@app.register_task +def add(a: int, b: int) -> int: + return a + b diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/some_tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/some_tasks.py new file mode 100644 index 0000000..e69de29 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py new file mode 100644 index 0000000..5fccda5 --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py @@ -0,0 +1,6 @@ +from . import some_file_name +from . import app +from . import some_module +from . import some_other_module + +__all__ = ["app", "some_file_name", "some_module", "some_other_module"] diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py new file mode 100644 index 0000000..905a39f --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py @@ -0,0 +1,8 @@ +"""A sample module that contains an app instance that encapsulates all app logic.""" + + +class SomeSimpleApp: + pass + + +some_simple_app = SomeSimpleApp() diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py new file mode 100644 index 0000000..cd25013 --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py @@ -0,0 +1,14 @@ +import time + +from aiotaskq.app import Aiotaskq + +app = Aiotaskq( + # includes=[".tasks"], +) +some_app = app + + +@app.register_task +def add(ls: list[int]) -> int: + time.sleep(5) + return sum(x for x in ls) diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py new file mode 100644 index 0000000..2b2f5de --- /dev/null +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py @@ -0,0 +1,6 @@ +from ..some_file_name import some_app + + +@some_app.register_task +def add(a: int, b: int) -> int: + return a + b diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/some_tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/some_tasks.py new file mode 100644 index 0000000..e69de29 diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py new file mode 100644 index 0000000..3a1047e --- /dev/null +++ b/src/aiotaskq/tests/test_app.py @@ -0,0 +1,60 @@ +from types import ModuleType +import typing as t + +import pytest + +import aiotaskq +from aiotaskq.task import Task +from aiotaskq.app import Aiotaskq + + +@pytest.mark.parametrize( + "valid_import_path,app_expected", + [ + ( + # Case 1: Tasks are defined inside a module without using explicit Aiotaskq + # instance, and we're instantiating Aiotaskq using the module path. + "aiotaskq.tests.apps.simple_app", + aiotaskq.tests.apps.simple_app, + ), + ( + # Case 2: Tasks are defined using an explicit Aiotaskq instance, and we're + # instantiating Aiotaskq from the import path to the instance. + "aiotaskq.tests.apps.simple_app_encapsulated", + aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, + ), + ( + # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default + # file name ("aiotaskq.py"), and we're instantiating Aiotaskq from the import + # path to the instance using the ":" pattern (this pattern is useful to + # differentiate between a module and an object). + "aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq:app", + aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, + ), + ( + # Case 4: Tasks are defined using an explicit Aiotaskq instance in a non-default + # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the + # import path to the instance using the ":" pattern (this pattern is useful to + # differentiate between a module and an object)". + "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name:some_app", + aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + ), + ( + # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default + # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the + # import path to the instance using the ":" pattern (this pattern is useful to + # differentiate between a module and an object)". + "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name", + aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + ), + ], +) +def test_valid_app_import_path(valid_import_path: str, app_expected: t.Union[Task, ModuleType]): + # Given a valid import path + + # When instantiating an Aiotaskq from the import path + app_actual = Aiotaskq.from_import_path(valid_import_path) + + # Then the instantiated Aiotaskq object should be loaded with the tasks + assert isinstance(app_actual.task_map["add"], Task) + assert app_expected.add == app_actual.task_map["add"] == app_actual.add # type: ignore diff --git a/src/aiotaskq/worker.py b/src/aiotaskq/worker.py index 2a85fd2..40104d7 100755 --- a/src/aiotaskq/worker.py +++ b/src/aiotaskq/worker.py @@ -11,8 +11,8 @@ import signal import sys import typing as t -import types +from .app import Aiotaskq from .concurrency_manager import ConcurrencyManagerSingleton from .constants import REDIS_URL, RESULTS_CHANNEL_TEMPLATE, TASKS_CHANNEL from .interfaces import ConcurrencyType, IConcurrencyManager, IPubSub @@ -31,12 +31,12 @@ class BaseWorker(ABC): where you write your `while True: some_logic`. """ - app: types.ModuleType + app: Aiotaskq pubsub: IPubSub concurrency_manager: IConcurrencyManager def __init__(self, app_import_path: str): - self.app = importlib.import_module(app_import_path) + self.app = Aiotaskq.from_import_path(app_or_module_path=app_import_path) def run_forever(self) -> None: """ @@ -166,7 +166,7 @@ async def _main_loop(self): def _start_grunt_workers(self): def _run_grunt_worker_forever(): grunt_worker = GruntWorker( - app_import_path=self.app.__name__, + app_import_path=self.app.import_path, poll_interval_s=self._poll_interval_s, ) grunt_worker.run_forever() diff --git a/test.sh b/test.sh index 5fa3b6b..3f15cef 100755 --- a/test.sh +++ b/test.sh @@ -1,13 +1,10 @@ -pip install --upgrade pip -pip install -e .[dev] - coverage erase if [ -z $1 ]; then coverage run -m pytest -v else - coverage run -m pytest -v -k $1 + coverage run -m pytest -vvv -k $1 fi failed=$? From 5cb18fd10dfd8cfd52b76d181e0ef2e67ec16fbf Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Thu, 17 Nov 2022 23:34:30 -0500 Subject: [PATCH 02/42] WIP --- .gitignore | 7 +- .vscode/launch.json | 8 +++ .vscode/settings.json | 3 +- pyproject.toml | 3 +- .../simple_app/pyproject.template.toml | 26 +++++++ .../simple_app/src/simple_app/aiotaskq.py | 6 ++ .../simple_app/src/simple_app/app_aiotaskq.py | 32 +++++++++ .../simple_app/src/simple_app/app_celery.py | 33 +++++++++ .../simple_app/src/simple_app}/celery.py | 2 +- .../src/simple_app/tasks_aiotaskq.py | 25 +++++++ .../simple_app/src/simple_app/tasks_celery.py | 27 ++++++++ .../simple_app/src/simple_app/tests.py | 0 src/aiotaskq/__init__.py | 19 ++---- src/aiotaskq/app.py | 21 +++++- .../tests/apps/simple_app_celery/app.py | 8 --- .../tests/apps/simple_app_celery/tasks.py | 32 --------- .../apps/simple_app_encapsulated/aiotaskq.py | 11 +-- src/aiotaskq/tests/test_app.py | 68 +++++++++---------- src/aiotaskq/worker.py | 3 + test.sh | 4 +- 20 files changed, 236 insertions(+), 102 deletions(-) create mode 100644 sample_apps/simple_app/pyproject.template.toml create mode 100644 sample_apps/simple_app/src/simple_app/aiotaskq.py create mode 100644 sample_apps/simple_app/src/simple_app/app_aiotaskq.py create mode 100644 sample_apps/simple_app/src/simple_app/app_celery.py rename {src/aiotaskq/tests/apps/simple_app_celery => sample_apps/simple_app/src/simple_app}/celery.py (75%) create mode 100644 sample_apps/simple_app/src/simple_app/tasks_aiotaskq.py create mode 100644 sample_apps/simple_app/src/simple_app/tasks_celery.py rename src/aiotaskq/tests/apps/simple_app_celery/__init__.py => sample_apps/simple_app/src/simple_app/tests.py (100%) delete mode 100644 src/aiotaskq/tests/apps/simple_app_celery/app.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_celery/tasks.py diff --git a/.gitignore b/.gitignore index e60bd8a..9764184 100644 --- a/.gitignore +++ b/.gitignore @@ -113,13 +113,18 @@ celerybeat.pid # Environments .env -.venv +.venv* env/ venv/ ENV/ env.bak/ venv.bak/ +# Sample Apps + +sample_apps/*/pyproject.toml +sample_apps/*/build/ + # Spyder project settings .spyderproject .spyproject diff --git a/.vscode/launch.json b/.vscode/launch.json index 0b568c3..63b0837 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,6 +19,14 @@ "args": [], "console": "integratedTerminal" }, + { + "name": "Sample App Celery", + "type": "python", + "request": "launch", + "module": "aiotaskq.tests.apps.simple_app_celery.app", + "args": [], + "console": "integratedTerminal" + }, { "name": "Test", "type": "python", diff --git a/.vscode/settings.json b/.vscode/settings.json index 9390569..03b59b2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,7 +11,8 @@ "python.testing.pytestEnabled": true, "python.testing.unittestEnabled": false, "python.analysis.extraPaths": [ - "src/tests/" + "src/tests/", + "./sample_apps" ], "python.defaultInterpreterPath": "../.venv/bin/python3" } diff --git a/pyproject.toml b/pyproject.toml index 9fe11e4..58b1f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ license = { file = "LICENSE" } [project.optional-dependencies] dev = [ "black >= 22.1.0, < 22.2.0", - "celery >= 5.2.0 < 5.3.0", + "celery >= 5.2.0, < 5.3.0", "coverage >= 6.4.0, < 6.5.0", "mypy >= 0.931, < 1.0", "mypy-extensions >= 0.4.0, < 0.5.0", @@ -37,6 +37,7 @@ dev = [ "pylint >= 2.14.0, < 2.15.0", "pytest >= 7.1.0, < 7.2.0", "typing_extensions >= 4.1.1, < 4.2.0", + "simple_app @ file://./sample_apps/simple_app", ] [project.urls] diff --git a/sample_apps/simple_app/pyproject.template.toml b/sample_apps/simple_app/pyproject.template.toml new file mode 100644 index 0000000..e06a8cb --- /dev/null +++ b/sample_apps/simple_app/pyproject.template.toml @@ -0,0 +1,26 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" + +[project] +requires-python = ">=3.9" +dependencies = [ + "celery >= 5.2.0, < 5.3.0", + "aiotaskq @ file://${PROJECT_DIR}/", +] +name = "simple_app" +version = "0.0.0" +readme = "README.md" +description = "A simple app using Celery" +authors = [ + {name = "Imran Ariffin", email = "ariffin.imran@gmail.com"}, +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +license = { file = "LICENSE" } diff --git a/sample_apps/simple_app/src/simple_app/aiotaskq.py b/sample_apps/simple_app/src/simple_app/aiotaskq.py new file mode 100644 index 0000000..8bd66b5 --- /dev/null +++ b/sample_apps/simple_app/src/simple_app/aiotaskq.py @@ -0,0 +1,6 @@ +from aiotaskq import Aiotaskq + + +app = Aiotaskq( + include=["simple_app.tasks_aiotaskq"], +) diff --git a/sample_apps/simple_app/src/simple_app/app_aiotaskq.py b/sample_apps/simple_app/src/simple_app/app_aiotaskq.py new file mode 100644 index 0000000..27c25b7 --- /dev/null +++ b/sample_apps/simple_app/src/simple_app/app_aiotaskq.py @@ -0,0 +1,32 @@ +import asyncio +import logging + +from simple_app.tasks_aiotaskq import add, times + +logger = logging.getLogger(__name__) + + +async def get_formula(): + """ + Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). + # TODO (Issue #44): Support chain of tasks + """ + logger.info("get_formula() ...") + x = await times.apply_async(x=1, y=3) + y = await times.apply_async(x=3, y=4) + ret = await add.apply_async([x, y]) + logger.info("get_formula() -> %s", str(ret)) + return ret + + +async def main(): + logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger("simple_app_aiotaskq.app") + logger.info("Simple App (Aiotaskq)") + ret = await get_formula() + logger.info("Result: %s", ret) + assert ret == 15 + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sample_apps/simple_app/src/simple_app/app_celery.py b/sample_apps/simple_app/src/simple_app/app_celery.py new file mode 100644 index 0000000..06a9376 --- /dev/null +++ b/sample_apps/simple_app/src/simple_app/app_celery.py @@ -0,0 +1,33 @@ +import logging + +from celery.canvas import Signature, chord, group +from simple_app.tasks_celery import add, times + +logger = logging.getLogger(__name__) + + +def get_formula() -> Signature: + """ + Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). + """ + logger.info("get_formula() ...") + times_tasks: list[Signature] = [ + times.si(x=1, y=3), + times.si(x=3, y=4), + ] + ret: Signature = chord(header=group(times_tasks), body=add.s()) + logger.info("get_formula() -> %s", str(ret)) + return ret + + +def main(): + logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger("simple_app_celery.app") + logger.info("Simple App (Celery)") + ret = get_formula().apply_async().get() + logger.info("Result: %s", ret) + assert ret == 15 + + +if __name__ == "__main__": + main() diff --git a/src/aiotaskq/tests/apps/simple_app_celery/celery.py b/sample_apps/simple_app/src/simple_app/celery.py similarity index 75% rename from src/aiotaskq/tests/apps/simple_app_celery/celery.py rename to sample_apps/simple_app/src/simple_app/celery.py index fd4311b..a597844 100644 --- a/src/aiotaskq/tests/apps/simple_app_celery/celery.py +++ b/sample_apps/simple_app/src/simple_app/celery.py @@ -2,7 +2,7 @@ app = Celery( - include=["aiotaskq.tests.apps.simple_app_celery.tasks"], + include=["simple_app.tasks_celery"], ) app.conf.broker_url = 'redis://localhost:6379/0' app.conf.result_backend = 'redis://localhost:6379/0' diff --git a/sample_apps/simple_app/src/simple_app/tasks_aiotaskq.py b/sample_apps/simple_app/src/simple_app/tasks_aiotaskq.py new file mode 100644 index 0000000..0948ea4 --- /dev/null +++ b/sample_apps/simple_app/src/simple_app/tasks_aiotaskq.py @@ -0,0 +1,25 @@ +import asyncio +import logging +import time + +from aiotaskq.task import task + +logger = logging.getLogger(__name__) + + +@task +def add(ls: list[int]) -> int: + logger.info("add(%s) ...", ls) + time.sleep(5) + ret = sum(x for x in ls) + logger.info("add(%s) -> %s", ls, ret) + return ret + + +@task +def times(x: int, y: int) -> int: + logger.info("times(%s, %s) ...", x, y) + time.sleep(2) + ret = x * y + logger.info("times(%s, %s) -> %s", x, y, ret) + return ret diff --git a/sample_apps/simple_app/src/simple_app/tasks_celery.py b/sample_apps/simple_app/src/simple_app/tasks_celery.py new file mode 100644 index 0000000..e2924b4 --- /dev/null +++ b/sample_apps/simple_app/src/simple_app/tasks_celery.py @@ -0,0 +1,27 @@ +import logging +import time + +from celery.canvas import chord, group +import celery + +from simple_app.celery import app + +logger = logging.getLogger(__name__) + + +@app.task() +def add(ls: list[int]) -> int: + logger.info("add(%s) ...", ls) + time.sleep(5) + ret = sum(x for x in ls) + logger.info("add(%s) -> %s", ls, ret) + return ret + + +@app.task() +def times(x: int, y: int) -> int: + logger.info("times(%s, %s) ...", x, y) + time.sleep(2) + ret = x * y + logger.info("times(%s, %s) -> %s", x, y, ret) + return ret diff --git a/src/aiotaskq/tests/apps/simple_app_celery/__init__.py b/sample_apps/simple_app/src/simple_app/tests.py similarity index 100% rename from src/aiotaskq/tests/apps/simple_app_celery/__init__.py rename to sample_apps/simple_app/src/simple_app/tests.py diff --git a/src/aiotaskq/__init__.py b/src/aiotaskq/__init__.py index 4709471..baa8f58 100644 --- a/src/aiotaskq/__init__.py +++ b/src/aiotaskq/__init__.py @@ -27,20 +27,15 @@ async def main(): """ -import os -from pathlib import Path -from typing import TYPE_CHECKING - -import tomlkit +import importlib +from .app import Aiotaskq from .task import task -with Path("pyproject.toml").open("r", encoding="utf-8") as f: - toml_dict = tomlkit.loads(f.read()) - __version__ = toml_dict["project"]["version"] +__version__ = importlib.metadata.version("aiotaskq") -__all__ = ["__version__", "task"] +__all__ = ["__version__", "task", "Aiotaskq"] -# if os.environ["ENV"] == "test" or TYPE_CHECKING: -from . import tests -__all__ += ["tests"] +# # if os.environ["ENV"] == "test" or TYPE_CHECKING: +# from . import tests +# __all__ += ["tests"] diff --git a/src/aiotaskq/app.py b/src/aiotaskq/app.py index 63c8c54..da4d409 100644 --- a/src/aiotaskq/app.py +++ b/src/aiotaskq/app.py @@ -1,5 +1,6 @@ import importlib import inspect +from types import ModuleType import typing as t from .exceptions import AppImportError @@ -12,13 +13,24 @@ class Aiotaskq: import_path: str task_map: dict[str, Task] = {} - def __init__(self, import_path: t.Optional[str] = None, tasks: t.Optional[list[Task]] = None): + def __init__( + self, + import_path: t.Optional[str] = None, + tasks: t.Optional[list[Task]] = None, + include: t.Optional[list[str]] = None, + ): self.import_path = ( import_path if import_path is not None else inspect.getmodule(self).__name__ # type: ignore ) self.task_map = {task_.__name__: task_ for task_ in tasks} if tasks else {} + if include is not None: + for module_path in include: + module = importlib.import_module(module_path) + tasks = self._extract_tasks(app_or_module=module) + print(f"module {module}, tasks: {tasks}") + self.task_map.update({task_.__name__: task_ for task_ in tasks}) def __getattribute__(self, attr_name: str) -> t.Any: try: @@ -62,7 +74,10 @@ def from_import_path(cls, app_or_module_path: str) -> "Aiotaskq": # Import path points to an Aiotaskq instance -- use it. print("\n1") app = app_or_module - elif hasattr(app_or_module, "app") and isinstance(getattr(app_or_module, "app"), Aiotaskq): + elif ( + hasattr(app_or_module, "app") + and isinstance(getattr(app_or_module, "app"), Aiotaskq) + ): # Import path points to a module that contains an Aiotaskq instance # named as `app` -- retrieve the instance and use it. print("\n2") @@ -100,7 +115,7 @@ def register_task(self, func: t.Callable[P, RT]) -> Task[P, RT]: return task_ @classmethod - def _extract_tasks(cls, app_or_module: t.Any) -> list[Task]: + def _extract_tasks(cls, app_or_module: "Aiotaskq | ModuleType") -> list[Task]: return [ getattr(app_or_module, attr) for attr in app_or_module.__dict__ diff --git a/src/aiotaskq/tests/apps/simple_app_celery/app.py b/src/aiotaskq/tests/apps/simple_app_celery/app.py deleted file mode 100644 index 1b74d1c..0000000 --- a/src/aiotaskq/tests/apps/simple_app_celery/app.py +++ /dev/null @@ -1,8 +0,0 @@ -from aiotaskq.tests.apps.simple_app_celery.tasks import get_formula -from .celery import app - - -if __name__ == "__main__": - formula = get_formula() - ret = formula.apply_async().get() - assert ret == 14 diff --git a/src/aiotaskq/tests/apps/simple_app_celery/tasks.py b/src/aiotaskq/tests/apps/simple_app_celery/tasks.py deleted file mode 100644 index 11f4699..0000000 --- a/src/aiotaskq/tests/apps/simple_app_celery/tasks.py +++ /dev/null @@ -1,32 +0,0 @@ -import time -from celery import chord, group -import celery - -from .celery import app - - -@app.task() -def add(ls: list[int]) -> int: - time.sleep(5) - return sum(x for x in ls) - - -@app.task() -def times(x: int, y: int) -> int: - time.sleep(2) - return x * y - - -def get_formula(): - """ - Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). - """ - - times_tasks: list[celery.Task] = [ - times.si(x=1, y=2), - times.si(x=3, y=4), - ] - return chord( - header=group(times_tasks), - body=add.s(), - ) diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py index 500d6e0..15436ed 100644 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py +++ b/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py @@ -1,12 +1,7 @@ -import time - from aiotaskq.app import Aiotaskq app = Aiotaskq( - # includes=[".tasks"], + # include=[ + # "aiotaskq.tests.apps.simple_app_encapsulated.some_module.tasks", + # ], ) - -@app.register_task -def add(ls: list[int]) -> int: - time.sleep(5) - return sum(x for x in ls) diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index 3a1047e..d06de81 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -1,11 +1,11 @@ from types import ModuleType -import typing as t import pytest import aiotaskq from aiotaskq.task import Task from aiotaskq.app import Aiotaskq +from simple_app.aiotaskq import app as simple_app @pytest.mark.parametrize( @@ -14,42 +14,42 @@ ( # Case 1: Tasks are defined inside a module without using explicit Aiotaskq # instance, and we're instantiating Aiotaskq using the module path. - "aiotaskq.tests.apps.simple_app", - aiotaskq.tests.apps.simple_app, - ), - ( - # Case 2: Tasks are defined using an explicit Aiotaskq instance, and we're - # instantiating Aiotaskq from the import path to the instance. - "aiotaskq.tests.apps.simple_app_encapsulated", - aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, - ), - ( - # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default - # file name ("aiotaskq.py"), and we're instantiating Aiotaskq from the import - # path to the instance using the ":" pattern (this pattern is useful to - # differentiate between a module and an object). - "aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq:app", - aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, - ), - ( - # Case 4: Tasks are defined using an explicit Aiotaskq instance in a non-default - # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the - # import path to the instance using the ":" pattern (this pattern is useful to - # differentiate between a module and an object)". - "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name:some_app", - aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, - ), - ( - # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default - # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the - # import path to the instance using the ":" pattern (this pattern is useful to - # differentiate between a module and an object)". - "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name", - aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + "simple_app.aiotaskq", + simple_app, ), + # ( + # # Case 2: Tasks are defined using an explicit Aiotaskq instance, and we're + # # instantiating Aiotaskq from the import path to the instance. + # "aiotaskq.tests.apps.simple_app_encapsulated", + # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, + # ), + # ( + # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default + # # file name ("aiotaskq.py"), and we're instantiating Aiotaskq from the import + # # path to the instance using the ":" pattern (this pattern is useful to + # # differentiate between a module and an object). + # "aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq:app", + # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, + # ), + # ( + # # Case 4: Tasks are defined using an explicit Aiotaskq instance in a non-default + # # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the + # # import path to the instance using the ":" pattern (this pattern is useful to + # # differentiate between a module and an object)". + # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name:some_app", + # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + # ), + # ( + # # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default + # # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the + # # import path to the instance using the ":" pattern (this pattern is useful to + # # differentiate between a module and an object)". + # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name", + # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + # ), ], ) -def test_valid_app_import_path(valid_import_path: str, app_expected: t.Union[Task, ModuleType]): +def test_valid_app_import_path(valid_import_path: str, app_expected: ModuleType): # Given a valid import path # When instantiating an Aiotaskq from the import path diff --git a/src/aiotaskq/worker.py b/src/aiotaskq/worker.py index 40104d7..22a6a78 100755 --- a/src/aiotaskq/worker.py +++ b/src/aiotaskq/worker.py @@ -118,6 +118,9 @@ def __init__( ) self._poll_interval_s = poll_interval_s super().__init__(app_import_path=app_import_path) + self._logger.debug("Tasks list:") + for task in self.app.task_map.keys(): + self._logger.debug("* %s", task) async def _pre_run(self): self._logger.info("Starting %s back workers", self.concurrency_manager.concurrency) diff --git a/test.sh b/test.sh index 3f15cef..3a33325 100755 --- a/test.sh +++ b/test.sh @@ -1,10 +1,12 @@ +PROJECT_DIR=$PWD envsubst < ./sample_apps/simple_app/pyproject.template.toml > ./sample_apps/simple_app/pyproject.toml + coverage erase if [ -z $1 ]; then coverage run -m pytest -v else - coverage run -m pytest -vvv -k $1 + coverage run -m pytest -vvv -k $1 -s fi failed=$? From 5bb9b4c231856d8f04d8fe2e9454a526f73c4d9d Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 10:07:32 -0500 Subject: [PATCH 03/42] Fixup --- src/aiotaskq/tests/test_app.py | 32 ++++++++++++++++++-------------- test.sh | 9 ++++++++- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index d06de81..69bd3d5 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -13,38 +13,42 @@ [ ( # Case 1: Tasks are defined inside a module without using explicit Aiotaskq - # instance, and we're instantiating Aiotaskq using the module path. + # instance, and "simple_app.aiotaskq", + # We should instantiate Aiotaskq using the module path. simple_app, ), # ( - # # Case 2: Tasks are defined using an explicit Aiotaskq instance, and we're - # # instantiating Aiotaskq from the import path to the instance. + # # Case 2: Tasks are defined using an explicit Aiotaskq instance, and # "aiotaskq.tests.apps.simple_app_encapsulated", + # # We should instantiate Aiotaskq from the import path to the instance. # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, # ), # ( # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default - # # file name ("aiotaskq.py"), and we're instantiating Aiotaskq from the import - # # path to the instance using the ":" pattern (this pattern is useful to - # # differentiate between a module and an object). + # # file name ("aiotaskq.py"), and # "aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq:app", + # # We should instantiate Aiotaskq from the import path to the instance + # # using the ":" pattern (this pattern is useful to differentiate between + # # a module and an object). # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, # ), # ( - # # Case 4: Tasks are defined using an explicit Aiotaskq instance in a non-default - # # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the - # # import path to the instance using the ":" pattern (this pattern is useful to - # # differentiate between a module and an object)". + # # Case 4: Tasks are defined using an explicit Aiotaskq instance in + # # a non-default file name ("some_module_name.py"), and # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name:some_app", + # # We should instantiate Aiotaskq from the import path to the instance + # # using the ":" pattern (this pattern is useful to differentiate between + # # a module and an object)". # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, # ), # ( # # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default - # # file name ("some_module_name.py"), and we're instantiating Aiotaskq from the - # # import path to the instance using the ":" pattern (this pattern is useful to - # # differentiate between a module and an object)". + # # file name ("some_module_name.py"), and # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name", + # # We should instantiate Aiotaskq from the import path to the instance + # # using the ":" pattern (this pattern is useful to differentiate between + # # a module and an object)". # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, # ), ], @@ -56,5 +60,5 @@ def test_valid_app_import_path(valid_import_path: str, app_expected: ModuleType) app_actual = Aiotaskq.from_import_path(valid_import_path) # Then the instantiated Aiotaskq object should be loaded with the tasks - assert isinstance(app_actual.task_map["add"], Task) + assert isinstance(app_actual.add, Task) assert app_expected.add == app_actual.task_map["add"] == app_actual.add # type: ignore diff --git a/test.sh b/test.sh index 3a33325..46a2e40 100755 --- a/test.sh +++ b/test.sh @@ -1,16 +1,23 @@ +echo "Upgrade pip" +pip install --quiet --upgrade pip + +echo "Install sample projects" PROJECT_DIR=$PWD envsubst < ./sample_apps/simple_app/pyproject.template.toml > ./sample_apps/simple_app/pyproject.toml +pip install --quiet -e ./sample_apps/simple_app/ +echo "Erase previous coverage files" coverage erase +echo "Run tests" if [ -z $1 ]; then coverage run -m pytest -v else coverage run -m pytest -vvv -k $1 -s fi - failed=$? +echo "Combine coverage files" coverage combine exit $failed From 82e56da0c9108480c50f576105d622725ecb44ca Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 11:18:19 -0500 Subject: [PATCH 04/42] Move out sample_apps to its own package: Passed tests --- pyproject.toml | 1 - .../{simple_app => }/pyproject.template.toml | 4 +-- sample_apps/pyproject.toml | 26 +++++++++++++++++++ .../simple_app/src/simple_app/aiotaskq.py | 6 ----- sample_apps/src/sample_apps/__init__.py | 5 ++++ .../src/sample_apps/simple_app/__init__.py | 17 ++++++++++++ .../src/sample_apps/simple_app/aiotaskq.py | 6 +++++ .../sample_apps}/simple_app/app_aiotaskq.py | 2 +- .../sample_apps}/simple_app/app_celery.py | 2 +- .../sample_apps}/simple_app/celery.py | 2 +- .../sample_apps}/simple_app/tasks_aiotaskq.py | 0 .../sample_apps}/simple_app/tasks_celery.py | 2 +- .../sample_apps}/simple_app/tests.py | 0 src/aiotaskq/tests/test_app.py | 5 ++-- test.sh | 13 ++++++---- 15 files changed, 71 insertions(+), 20 deletions(-) rename sample_apps/{simple_app => }/pyproject.template.toml (84%) create mode 100644 sample_apps/pyproject.toml delete mode 100644 sample_apps/simple_app/src/simple_app/aiotaskq.py create mode 100644 sample_apps/src/sample_apps/__init__.py create mode 100644 sample_apps/src/sample_apps/simple_app/__init__.py create mode 100644 sample_apps/src/sample_apps/simple_app/aiotaskq.py rename sample_apps/{simple_app/src => src/sample_apps}/simple_app/app_aiotaskq.py (93%) rename sample_apps/{simple_app/src => src/sample_apps}/simple_app/app_celery.py (94%) rename sample_apps/{simple_app/src => src/sample_apps}/simple_app/celery.py (78%) rename sample_apps/{simple_app/src => src/sample_apps}/simple_app/tasks_aiotaskq.py (100%) rename sample_apps/{simple_app/src => src/sample_apps}/simple_app/tasks_celery.py (93%) rename sample_apps/{simple_app/src => src/sample_apps}/simple_app/tests.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 58b1f94..0e425b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ dev = [ "pylint >= 2.14.0, < 2.15.0", "pytest >= 7.1.0, < 7.2.0", "typing_extensions >= 4.1.1, < 4.2.0", - "simple_app @ file://./sample_apps/simple_app", ] [project.urls] diff --git a/sample_apps/simple_app/pyproject.template.toml b/sample_apps/pyproject.template.toml similarity index 84% rename from sample_apps/simple_app/pyproject.template.toml rename to sample_apps/pyproject.template.toml index e06a8cb..26f631a 100644 --- a/sample_apps/simple_app/pyproject.template.toml +++ b/sample_apps/pyproject.template.toml @@ -11,10 +11,10 @@ dependencies = [ "celery >= 5.2.0, < 5.3.0", "aiotaskq @ file://${PROJECT_DIR}/", ] -name = "simple_app" +name = "sample_apps" version = "0.0.0" readme = "README.md" -description = "A simple app using Celery" +description = "A collection of sample apps to test aiotaskq against Celery" authors = [ {name = "Imran Ariffin", email = "ariffin.imran@gmail.com"}, ] diff --git a/sample_apps/pyproject.toml b/sample_apps/pyproject.toml new file mode 100644 index 0000000..784f7a5 --- /dev/null +++ b/sample_apps/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" + +[project] +requires-python = ">=3.9" +dependencies = [ + "celery >= 5.2.0, < 5.3.0", + "aiotaskq @ file:///home/in-gote/workspace/aiotaskq/", +] +name = "sample_apps" +version = "0.0.0" +readme = "README.md" +description = "A collection of sample apps to test aiotaskq against Celery" +authors = [ + {name = "Imran Ariffin", email = "ariffin.imran@gmail.com"}, +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +license = { file = "LICENSE" } diff --git a/sample_apps/simple_app/src/simple_app/aiotaskq.py b/sample_apps/simple_app/src/simple_app/aiotaskq.py deleted file mode 100644 index 8bd66b5..0000000 --- a/sample_apps/simple_app/src/simple_app/aiotaskq.py +++ /dev/null @@ -1,6 +0,0 @@ -from aiotaskq import Aiotaskq - - -app = Aiotaskq( - include=["simple_app.tasks_aiotaskq"], -) diff --git a/sample_apps/src/sample_apps/__init__.py b/sample_apps/src/sample_apps/__init__.py new file mode 100644 index 0000000..7c46f98 --- /dev/null +++ b/sample_apps/src/sample_apps/__init__.py @@ -0,0 +1,5 @@ +from . import simple_app + +__all__ = [ + "simple_app", +] diff --git a/sample_apps/src/sample_apps/simple_app/__init__.py b/sample_apps/src/sample_apps/simple_app/__init__.py new file mode 100644 index 0000000..056d109 --- /dev/null +++ b/sample_apps/src/sample_apps/simple_app/__init__.py @@ -0,0 +1,17 @@ +from . import ( + aiotaskq, + app_aiotaskq, + app_celery, + celery, + tasks_aiotaskq, + tasks_celery, +) + +__all__ = [ + "aiotaskq", + "app_aiotaskq", + "app_celery", + "celery", + "tasks_aiotaskq", + "tasks_celery", +] diff --git a/sample_apps/src/sample_apps/simple_app/aiotaskq.py b/sample_apps/src/sample_apps/simple_app/aiotaskq.py new file mode 100644 index 0000000..fddceb4 --- /dev/null +++ b/sample_apps/src/sample_apps/simple_app/aiotaskq.py @@ -0,0 +1,6 @@ +from aiotaskq import Aiotaskq + + +app = Aiotaskq( + include=["sample_apps.simple_app.tasks_aiotaskq"], +) diff --git a/sample_apps/simple_app/src/simple_app/app_aiotaskq.py b/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py similarity index 93% rename from sample_apps/simple_app/src/simple_app/app_aiotaskq.py rename to sample_apps/src/sample_apps/simple_app/app_aiotaskq.py index 27c25b7..a016dd8 100644 --- a/sample_apps/simple_app/src/simple_app/app_aiotaskq.py +++ b/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py @@ -1,7 +1,7 @@ import asyncio import logging -from simple_app.tasks_aiotaskq import add, times +from .tasks_aiotaskq import add, times logger = logging.getLogger(__name__) diff --git a/sample_apps/simple_app/src/simple_app/app_celery.py b/sample_apps/src/sample_apps/simple_app/app_celery.py similarity index 94% rename from sample_apps/simple_app/src/simple_app/app_celery.py rename to sample_apps/src/sample_apps/simple_app/app_celery.py index 06a9376..5115253 100644 --- a/sample_apps/simple_app/src/simple_app/app_celery.py +++ b/sample_apps/src/sample_apps/simple_app/app_celery.py @@ -1,7 +1,7 @@ import logging from celery.canvas import Signature, chord, group -from simple_app.tasks_celery import add, times +from .tasks_celery import add, times logger = logging.getLogger(__name__) diff --git a/sample_apps/simple_app/src/simple_app/celery.py b/sample_apps/src/sample_apps/simple_app/celery.py similarity index 78% rename from sample_apps/simple_app/src/simple_app/celery.py rename to sample_apps/src/sample_apps/simple_app/celery.py index a597844..c590727 100644 --- a/sample_apps/simple_app/src/simple_app/celery.py +++ b/sample_apps/src/sample_apps/simple_app/celery.py @@ -2,7 +2,7 @@ app = Celery( - include=["simple_app.tasks_celery"], + include=["sample_apps.simple_app.tasks_celery"], ) app.conf.broker_url = 'redis://localhost:6379/0' app.conf.result_backend = 'redis://localhost:6379/0' diff --git a/sample_apps/simple_app/src/simple_app/tasks_aiotaskq.py b/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py similarity index 100% rename from sample_apps/simple_app/src/simple_app/tasks_aiotaskq.py rename to sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py diff --git a/sample_apps/simple_app/src/simple_app/tasks_celery.py b/sample_apps/src/sample_apps/simple_app/tasks_celery.py similarity index 93% rename from sample_apps/simple_app/src/simple_app/tasks_celery.py rename to sample_apps/src/sample_apps/simple_app/tasks_celery.py index e2924b4..3fb6faa 100644 --- a/sample_apps/simple_app/src/simple_app/tasks_celery.py +++ b/sample_apps/src/sample_apps/simple_app/tasks_celery.py @@ -4,7 +4,7 @@ from celery.canvas import chord, group import celery -from simple_app.celery import app +from .celery import app logger = logging.getLogger(__name__) diff --git a/sample_apps/simple_app/src/simple_app/tests.py b/sample_apps/src/sample_apps/simple_app/tests.py similarity index 100% rename from sample_apps/simple_app/src/simple_app/tests.py rename to sample_apps/src/sample_apps/simple_app/tests.py diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index 69bd3d5..2bd2249 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -5,7 +5,8 @@ import aiotaskq from aiotaskq.task import Task from aiotaskq.app import Aiotaskq -from simple_app.aiotaskq import app as simple_app + +from sample_apps.simple_app.aiotaskq import app as simple_app @pytest.mark.parametrize( @@ -14,7 +15,7 @@ ( # Case 1: Tasks are defined inside a module without using explicit Aiotaskq # instance, and - "simple_app.aiotaskq", + "sample_apps.simple_app.aiotaskq", # We should instantiate Aiotaskq using the module path. simple_app, ), diff --git a/test.sh b/test.sh index 46a2e40..bdf919e 100755 --- a/test.sh +++ b/test.sh @@ -1,9 +1,12 @@ -echo "Upgrade pip" -pip install --quiet --upgrade pip +# echo "Upgrade pip" +# pip install --quiet --upgrade pip -echo "Install sample projects" -PROJECT_DIR=$PWD envsubst < ./sample_apps/simple_app/pyproject.template.toml > ./sample_apps/simple_app/pyproject.toml -pip install --quiet -e ./sample_apps/simple_app/ +# echo "Install dependencies" +# pip install -v --no-cache-dir -e .[dev] + +echo "Install sample apps" +PROJECT_DIR=$PWD envsubst < ./sample_apps/pyproject.template.toml > ./sample_apps/pyproject.toml +pip install --quiet --no-cache-dir -e file://$PWD/sample_apps/ echo "Erase previous coverage files" coverage erase From 74f6d7a9a8d2cc3074a84ea2350a59d229290bb7 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 11:21:06 -0500 Subject: [PATCH 05/42] Move sample_apps under root/src: Passed tests --- {sample_apps/src => src}/sample_apps/__init__.py | 0 .../sample_apps}/pyproject.template.toml | 0 {sample_apps => src/sample_apps}/pyproject.toml | 0 .../src => src}/sample_apps/simple_app/__init__.py | 0 .../src => src}/sample_apps/simple_app/aiotaskq.py | 0 .../sample_apps/simple_app/app_aiotaskq.py | 0 .../src => src}/sample_apps/simple_app/app_celery.py | 0 .../src => src}/sample_apps/simple_app/celery.py | 0 .../sample_apps/simple_app/tasks_aiotaskq.py | 0 .../sample_apps/simple_app/tasks_celery.py | 0 .../src => src}/sample_apps/simple_app/tests.py | 0 test.sh | 12 ++++++------ 12 files changed, 6 insertions(+), 6 deletions(-) rename {sample_apps/src => src}/sample_apps/__init__.py (100%) rename {sample_apps => src/sample_apps}/pyproject.template.toml (100%) rename {sample_apps => src/sample_apps}/pyproject.toml (100%) rename {sample_apps/src => src}/sample_apps/simple_app/__init__.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/aiotaskq.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/app_aiotaskq.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/app_celery.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/celery.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/tasks_aiotaskq.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/tasks_celery.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/tests.py (100%) diff --git a/sample_apps/src/sample_apps/__init__.py b/src/sample_apps/__init__.py similarity index 100% rename from sample_apps/src/sample_apps/__init__.py rename to src/sample_apps/__init__.py diff --git a/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml similarity index 100% rename from sample_apps/pyproject.template.toml rename to src/sample_apps/pyproject.template.toml diff --git a/sample_apps/pyproject.toml b/src/sample_apps/pyproject.toml similarity index 100% rename from sample_apps/pyproject.toml rename to src/sample_apps/pyproject.toml diff --git a/sample_apps/src/sample_apps/simple_app/__init__.py b/src/sample_apps/simple_app/__init__.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/__init__.py rename to src/sample_apps/simple_app/__init__.py diff --git a/sample_apps/src/sample_apps/simple_app/aiotaskq.py b/src/sample_apps/simple_app/aiotaskq.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/aiotaskq.py rename to src/sample_apps/simple_app/aiotaskq.py diff --git a/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py b/src/sample_apps/simple_app/app_aiotaskq.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/app_aiotaskq.py rename to src/sample_apps/simple_app/app_aiotaskq.py diff --git a/sample_apps/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/simple_app/app_celery.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/app_celery.py rename to src/sample_apps/simple_app/app_celery.py diff --git a/sample_apps/src/sample_apps/simple_app/celery.py b/src/sample_apps/simple_app/celery.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/celery.py rename to src/sample_apps/simple_app/celery.py diff --git a/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py b/src/sample_apps/simple_app/tasks_aiotaskq.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py rename to src/sample_apps/simple_app/tasks_aiotaskq.py diff --git a/sample_apps/src/sample_apps/simple_app/tasks_celery.py b/src/sample_apps/simple_app/tasks_celery.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/tasks_celery.py rename to src/sample_apps/simple_app/tasks_celery.py diff --git a/sample_apps/src/sample_apps/simple_app/tests.py b/src/sample_apps/simple_app/tests.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/tests.py rename to src/sample_apps/simple_app/tests.py diff --git a/test.sh b/test.sh index bdf919e..88f8dcc 100755 --- a/test.sh +++ b/test.sh @@ -1,12 +1,12 @@ -# echo "Upgrade pip" -# pip install --quiet --upgrade pip +echo "Upgrade pip" +pip install --quiet --upgrade pip -# echo "Install dependencies" -# pip install -v --no-cache-dir -e .[dev] +echo "Install dependencies" +pip install -v --no-cache-dir -e .[dev] echo "Install sample apps" -PROJECT_DIR=$PWD envsubst < ./sample_apps/pyproject.template.toml > ./sample_apps/pyproject.toml -pip install --quiet --no-cache-dir -e file://$PWD/sample_apps/ +PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +pip install --quiet --no-cache-dir -e file://$PWD/src/sample_apps/ echo "Erase previous coverage files" coverage erase From 6ad672340dc06d38b41188e9dbf9f8247e045b9a Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 17:30:06 -0500 Subject: [PATCH 06/42] Ignore sample_apps/pyproject.toml, commited by mistake --- .gitignore | 11 +++++++++-- src/sample_apps/pyproject.toml | 26 -------------------------- 2 files changed, 9 insertions(+), 28 deletions(-) delete mode 100644 src/sample_apps/pyproject.toml diff --git a/.gitignore b/.gitignore index 9764184..cf5471c 100644 --- a/.gitignore +++ b/.gitignore @@ -122,8 +122,15 @@ venv.bak/ # Sample Apps -sample_apps/*/pyproject.toml -sample_apps/*/build/ +src/sample_apps/pyproject.toml +src/sample_apps/.Python +src/sample_apps/build/ +src/sample_apps/develop-eggs/ +src/sample_apps/dist/ +src/sample_apps/eggs/ +src/sample_apps/.eggs/ +src/sample_apps/*.egg-info/ +src/sample_apps/*.egg # Spyder project settings .spyderproject diff --git a/src/sample_apps/pyproject.toml b/src/sample_apps/pyproject.toml deleted file mode 100644 index 784f7a5..0000000 --- a/src/sample_apps/pyproject.toml +++ /dev/null @@ -1,26 +0,0 @@ -[build-system] -requires = [ - "setuptools>=42", - "wheel" -] -build-backend = "setuptools.build_meta" - -[project] -requires-python = ">=3.9" -dependencies = [ - "celery >= 5.2.0, < 5.3.0", - "aiotaskq @ file:///home/in-gote/workspace/aiotaskq/", -] -name = "sample_apps" -version = "0.0.0" -readme = "README.md" -description = "A collection of sample apps to test aiotaskq against Celery" -authors = [ - {name = "Imran Ariffin", email = "ariffin.imran@gmail.com"}, -] -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", -] -license = { file = "LICENSE" } From acd2abd19c169809961c107b8edc4db358bc1aa3 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 18:31:12 -0500 Subject: [PATCH 07/42] Add README.md for sample_apps --- .gitignore | 1 + src/sample_apps/README.md | 91 +++++++++++++++++++++++++ src/sample_apps/pyproject.template.toml | 3 +- 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 src/sample_apps/README.md diff --git a/.gitignore b/.gitignore index cf5471c..8ecc03e 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ venv.bak/ # Sample Apps +src/sample_apps/.venv/ src/sample_apps/pyproject.toml src/sample_apps/.Python src/sample_apps/build/ diff --git a/src/sample_apps/README.md b/src/sample_apps/README.md new file mode 100644 index 0000000..e6ef536 --- /dev/null +++ b/src/sample_apps/README.md @@ -0,0 +1,91 @@ +# Sample apps + +## Outline +* [List of sample apps](#list-of-sample-apps) +* [Run a sample app with aiotaskq](#run-a-sample-app-with-aiotaskq) +* [Run a sample app with Celery](#run-a-sample-app-with-celery) + +These are the sample apps that we use to test against and compare against +`Celery`. We define these sample apps in such a way that we can them both +using `aiotaskq` and `Celery` separately. This way we can compare them in terms +of performance, correctness, parity etc. This is also a way to document the +fact that `aiotaskq` is compatible with any framework that `Celery` supports, and +a recommended way to use `aiotaskq` alongside a specific framework. + +## List of sample apps + +- [x] [Simple App (No framework; explicit app instance)](/src/sample_apps/simple_app/) +- [ ] Simple App `FastAPI` +- [ ] Simple App `Django` +- [ ] Simple App `Starlette` +- [ ] Simple App `Starlite` +- [ ] Simple App `Sanic` +- [ ] Simple App `Quart` + +## Run a sample app with aiotaskq + +Feel free to run this inside of a `aiotaskq` repository. Copy-pasting +these commands to your terminal should work out of the box. + +```bash +# If not already inside it, clone the aiotaskq repository and cd into it +[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) + +# Let's say we want to run the first app (Simple App), which is located +# in `./sample_apps/simple_app/`. Update this env var APP as you'd like +# to choose your desired sample app. +APP=simple_app + +# Create a new virtual env specifically for the sample apps +python -m venv ./sample_apps/.venv +source ./sample_apps/.venv/bin/activate + +# Install sample_apps package from local file +pip install -e sample_apps + +# Run aiotaskq workers in background and wait for it be ready +aiotaskq worker sample_apps.$APP --concurrency 4 & +sleep 2 + +# Run the the sample app +python -m sample_apps.$APP.app_aiotaskq + +# Confirm in the logs if the app is running correctly + +# Kill aiotaskq workers that were running in background +ps | grep aiotaskq | cut -d ' ' -f 1 | xargs kill -TERM +``` + +## Run a sample app with Celery + +Now, if you want to run the sample app with `Celery` to see how `aiotaskq` +compares to it, do the following: + +```bash +# If not already inside it, clone the aiotaskq repository and cd into it +[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) + +# Let's say we want to run the first app (Simple App), which is located +# in `./sample_apps/simple_app/`. Update this env var APP as you'd like +# to choose your desired sample app. +APP=simple_app + +# Create a new virtual env specifically for the sample apps +python -m venv ./sample_apps/.venv +source ./sample_apps/.venv/bin/activate + +# Install sample_apps package from local file +pip install -e sample_apps + +# Run Celery workers in background and wait for it to be ready +celery -A sample_apps.$APP worker --concurrency 4 & +sleep 2 + +# Run the the sample app +python -m sample_apps.$APP.app_celery + +# Confirm in the logs if the app is running correctly + +# Kill celery workers that were running in background +ps | grep celery | cut -d ' ' -f 1 | xargs kill -TERM +``` diff --git a/src/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml index 26f631a..733177f 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -8,8 +8,9 @@ build-backend = "setuptools.build_meta" [project] requires-python = ">=3.9" dependencies = [ - "celery >= 5.2.0, < 5.3.0", "aiotaskq @ file://${PROJECT_DIR}/", + "celery >= 5.2.0, < 5.3.0", + "redis >= 4.3.4, < 4.3.0", ] name = "sample_apps" version = "0.0.0" From f16433b7de76a0f9971cfdf97a554f20a62ed0d3 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 18:33:15 -0500 Subject: [PATCH 08/42] Add link from README.md to src/sample_apps/README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3d8cb67..35db794 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ python ./app.py # Output: sync_result == async_result == 165580141. Awesome! ``` +For sample apps that you can run right away, see [Sample Apps](/src/sample_apps/README.md) + ## Advanced usage example Let's say we want to compose a workflow where we want to break up some of the tasks and run them in parallel: ``` From 6ee7ae17eb816be149330d5a7a0e3aa6413ea4a5 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 18:34:28 -0500 Subject: [PATCH 09/42] Update VS code extra_paths to include src/sample_apps/ --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 03b59b2..897c5b9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,7 @@ "python.testing.unittestEnabled": false, "python.analysis.extraPaths": [ "src/tests/", - "./sample_apps" + "./src/sample_apps" ], "python.defaultInterpreterPath": "../.venv/bin/python3" } From b186c685b7d75ea9699ac27f0b1f48600b29dc72 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 18:47:34 -0500 Subject: [PATCH 10/42] Fixup --- src/aiotaskq/tests/test_app.py | 30 +++++++++++++++++-------- src/sample_apps/pyproject.template.toml | 2 +- test.sh | 6 ++--- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index 2bd2249..eba4609 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -12,19 +12,31 @@ @pytest.mark.parametrize( "valid_import_path,app_expected", [ + # ( + # # Case 1: Tasks are defined inside a module without using explicit Aiotaskq + # # instance, and + # "sample_apps.simple_app.aiotaskq", + # # We should instantiate Aiotaskq using the module path. + # simple_app, + # ), + ( + # Case 2.1: Tasks are defined using an explicit Aiotaskq instance, and + "sample_apps.simple_app.aiotaskq:app", + # We should instantiate Aiotaskq from the import path to the instance. + simple_app, + ), ( - # Case 1: Tasks are defined inside a module without using explicit Aiotaskq - # instance, and + # Case 2.2: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq", - # We should instantiate Aiotaskq using the module path. + # We should instantiate Aiotaskq from the import path to the instance. + simple_app, + ), + ( + # Case 2.3: Tasks are defined using an explicit Aiotaskq instance, and + "sample_apps.simple_app", + # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), - # ( - # # Case 2: Tasks are defined using an explicit Aiotaskq instance, and - # "aiotaskq.tests.apps.simple_app_encapsulated", - # # We should instantiate Aiotaskq from the import path to the instance. - # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, - # ), # ( # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default # # file name ("aiotaskq.py"), and diff --git a/src/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml index 733177f..403951d 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -10,7 +10,7 @@ requires-python = ">=3.9" dependencies = [ "aiotaskq @ file://${PROJECT_DIR}/", "celery >= 5.2.0, < 5.3.0", - "redis >= 4.3.4, < 4.3.0", + "redis >= 4.3.4, < 4.4.0", ] name = "sample_apps" version = "0.0.0" diff --git a/test.sh b/test.sh index 88f8dcc..7d14973 100755 --- a/test.sh +++ b/test.sh @@ -1,12 +1,12 @@ echo "Upgrade pip" -pip install --quiet --upgrade pip +python -m pip install --quiet --upgrade pip echo "Install dependencies" -pip install -v --no-cache-dir -e .[dev] +pip install --quiet -e .[dev] echo "Install sample apps" PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install --quiet --no-cache-dir -e file://$PWD/src/sample_apps/ +pip install --quiet -e file://$PWD/src/sample_apps/ echo "Erase previous coverage files" coverage erase From 072956c0428155c68f646fed1ce73f438233bf92 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 18:53:29 -0500 Subject: [PATCH 11/42] Delint & fix broken black --- pyproject.toml | 2 +- src/aiotaskq/app.py | 24 +++++++++++++----------- src/aiotaskq/interfaces.py | 1 + src/aiotaskq/tests/test_app.py | 7 +++---- src/sample_apps/simple_app/__init__.py | 20 ++++++++++---------- src/sample_apps/simple_app/celery.py | 4 ++-- src/tests/apps/simple_app.py | 2 +- 7 files changed, 31 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0e425b5..ae0e7ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ license = { file = "LICENSE" } [project.optional-dependencies] dev = [ - "black >= 22.1.0, < 22.2.0", + "black >= 22.3.0, < 22.4.0", "celery >= 5.2.0, < 5.3.0", "coverage >= 6.4.0, < 6.5.0", "mypy >= 0.931, < 1.0", diff --git a/src/aiotaskq/app.py b/src/aiotaskq/app.py index da4d409..b32d064 100644 --- a/src/aiotaskq/app.py +++ b/src/aiotaskq/app.py @@ -1,11 +1,15 @@ +"""Define the logic to create an aiotaskq app instance.""" + import importlib import inspect -from types import ModuleType import typing as t from .exceptions import AppImportError from .task import task, Task, P, RT +if t.TYPE_CHECKING: # pragma: no cover + from types import ModuleType + class Aiotaskq: """Encapsulate the whole application logic.""" @@ -14,14 +18,14 @@ class Aiotaskq: task_map: dict[str, Task] = {} def __init__( - self, - import_path: t.Optional[str] = None, + self, + import_path: t.Optional[str] = None, tasks: t.Optional[list[Task]] = None, include: t.Optional[list[str]] = None, ): self.import_path = ( - import_path - if import_path is not None + import_path + if import_path is not None else inspect.getmodule(self).__name__ # type: ignore ) self.task_map = {task_.__name__: task_ for task_ in tasks} if tasks else {} @@ -74,17 +78,15 @@ def from_import_path(cls, app_or_module_path: str) -> "Aiotaskq": # Import path points to an Aiotaskq instance -- use it. print("\n1") app = app_or_module - elif ( - hasattr(app_or_module, "app") - and isinstance(getattr(app_or_module, "app"), Aiotaskq) + elif hasattr(app_or_module, "app") and isinstance( + getattr(app_or_module, "app"), Aiotaskq ): # Import path points to a module that contains an Aiotaskq instance # named as `app` -- retrieve the instance and use it. print("\n2") app = getattr(app_or_module, "app") - elif ( - hasattr(app_or_module, "aiotaskq") - and isinstance(getattr(app_or_module, "aiotaskq"), Aiotaskq) + elif hasattr(app_or_module, "aiotaskq") and isinstance( + getattr(app_or_module, "aiotaskq"), Aiotaskq ): # Import path points to a module that contains an Aiotaskq instance # named as `aiotaskq` -- retrieve the instance and use it. diff --git a/src/aiotaskq/interfaces.py b/src/aiotaskq/interfaces.py index 0e64490..69584a0 100644 --- a/src/aiotaskq/interfaces.py +++ b/src/aiotaskq/interfaces.py @@ -93,6 +93,7 @@ class IPubSub(t.Protocol): print(f"Got message: {message}") ``` """ + def __init__(self, url: str, poll_interval_s: float, *args, **kwargs): """Initialize the pubsub class.""" diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index eba4609..660dea0 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -2,7 +2,6 @@ import pytest -import aiotaskq from aiotaskq.task import Task from aiotaskq.app import Aiotaskq @@ -20,19 +19,19 @@ # simple_app, # ), ( - # Case 2.1: Tasks are defined using an explicit Aiotaskq instance, and + # Case 2.1: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq:app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 2.2: Tasks are defined using an explicit Aiotaskq instance, and + # Case 2.2: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 2.3: Tasks are defined using an explicit Aiotaskq instance, and + # Case 2.3: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, diff --git a/src/sample_apps/simple_app/__init__.py b/src/sample_apps/simple_app/__init__.py index 056d109..466fc67 100644 --- a/src/sample_apps/simple_app/__init__.py +++ b/src/sample_apps/simple_app/__init__.py @@ -1,17 +1,17 @@ from . import ( - aiotaskq, - app_aiotaskq, - app_celery, - celery, - tasks_aiotaskq, + aiotaskq, + app_aiotaskq, + app_celery, + celery, + tasks_aiotaskq, tasks_celery, ) __all__ = [ - "aiotaskq", - "app_aiotaskq", - "app_celery", - "celery", - "tasks_aiotaskq", + "aiotaskq", + "app_aiotaskq", + "app_celery", + "celery", + "tasks_aiotaskq", "tasks_celery", ] diff --git a/src/sample_apps/simple_app/celery.py b/src/sample_apps/simple_app/celery.py index c590727..d40ddf4 100644 --- a/src/sample_apps/simple_app/celery.py +++ b/src/sample_apps/simple_app/celery.py @@ -4,8 +4,8 @@ app = Celery( include=["sample_apps.simple_app.tasks_celery"], ) -app.conf.broker_url = 'redis://localhost:6379/0' -app.conf.result_backend = 'redis://localhost:6379/0' +app.conf.broker_url = "redis://localhost:6379/0" +app.conf.result_backend = "redis://localhost:6379/0" if __name__ == "__main__": app.start() diff --git a/src/tests/apps/simple_app.py b/src/tests/apps/simple_app.py index b85ba1d..e49b5ba 100644 --- a/src/tests/apps/simple_app.py +++ b/src/tests/apps/simple_app.py @@ -8,7 +8,7 @@ def add(x: int, y: int) -> int: @aiotaskq.task def power(a: int, b: int = 1) -> int: - return a ** b + return a**b @aiotaskq.task From 7075bcabea56bbeae48930748402add905aa25f1 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 18:59:56 -0500 Subject: [PATCH 12/42] Fix vscode launch.json --- .vscode/launch.json | 28 +++++++++++++++++----------- src/aiotaskq/worker.py | 8 ++++---- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 63b0837..9911585 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,23 +9,26 @@ "type": "python", "request": "launch", "program": "./src/aiotaskq/main.py", - "console": "integratedTerminal" + "console": "integratedTerminal", + "justMyCode": false }, { - "name": "Sample App", + "name": "Sample App (aiotaskq)", "type": "python", "request": "launch", - "module": "tests.apps.simple_app", + "module": "sample_apps.simple_app.app_aiotaskq", "args": [], - "console": "integratedTerminal" + "console": "integratedTerminal", + "justMyCode": false }, { - "name": "Sample App Celery", + "name": "Sample App (Celery)", "type": "python", "request": "launch", - "module": "aiotaskq.tests.apps.simple_app_celery.app", + "module": "sample_apps.simple_app.app_celery", "args": [], - "console": "integratedTerminal" + "console": "integratedTerminal", + "justMyCode": false }, { "name": "Test", @@ -42,26 +45,29 @@ "console": "integratedTerminal" }, { - "name": "Sample Worker (Simple App)", + "name": "Simple App Worker (aiotaskq)", "type": "python", "request": "launch", "module": "aiotaskq", "args": [ "worker", - "tests.apps.simple_app" + "sample_apps.simple_app.aiotaskq:app" ], "console": "integratedTerminal", + "justMyCode": false }, { - "name": "Sample Worker (Simple App Celery)", + "name": "Simple App Worker (Celery)", "type": "python", "request": "launch", "module": "celery", "args": [ "-A", - "aiotaskq.tests.apps.simple_app_celery", + "sample_apps.simple_app.celery:app", "worker", + "--loglevel=DEBUG", ], + "console": "integratedTerminal", "justMyCode": false } ] diff --git a/src/aiotaskq/worker.py b/src/aiotaskq/worker.py index 22a6a78..bc6a89f 100755 --- a/src/aiotaskq/worker.py +++ b/src/aiotaskq/worker.py @@ -3,7 +3,6 @@ from abc import ABC, abstractmethod import asyncio from functools import cached_property -import importlib import json import logging import multiprocessing @@ -15,6 +14,7 @@ from .app import Aiotaskq from .concurrency_manager import ConcurrencyManagerSingleton from .constants import REDIS_URL, RESULTS_CHANNEL_TEMPLATE, TASKS_CHANNEL +from .exceptions import AppImportError from .interfaces import ConcurrencyType, IConcurrencyManager, IPubSub from .pubsub import PubSubSingleton @@ -233,10 +233,10 @@ async def _main_loop(self): def validate_input(app_import_path: str) -> t.Optional[str]: """Validate all worker cli inputs and return an error string if any.""" try: - importlib.import_module(app_import_path) - except ModuleNotFoundError: + Aiotaskq.from_import_path(app_or_module_path=app_import_path) + except AppImportError: return ( - f"Error at argument `--app_import_path {app_import_path}`:" + f"Error at argument `APP`:" f' "{app_import_path}" is not a path to a valid Python module' ) From 838821e2b832f16461feddbc92b9d83ca45f9d8b Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 19 Nov 2022 19:48:10 -0500 Subject: [PATCH 13/42] Pass Case 1 & 2 --- src/aiotaskq/tests/test_app.py | 27 ++++++---- src/sample_apps/__init__.py | 3 +- src/sample_apps/pyproject.template.toml | 12 ++--- .../simple_app_implicit_instance/__init__.py | 3 ++ .../app_aiotaskq.py | 51 +++++++++++++++++++ src/tests/test_worker.py | 8 +-- 6 files changed, 79 insertions(+), 25 deletions(-) create mode 100644 src/sample_apps/simple_app_implicit_instance/__init__.py create mode 100644 src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index 660dea0..0192477 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -6,36 +6,37 @@ from aiotaskq.app import Aiotaskq from sample_apps.simple_app.aiotaskq import app as simple_app +from sample_apps.simple_app_implicit_instance import app_aiotaskq as simple_app_implicit_instance @pytest.mark.parametrize( "valid_import_path,app_expected", [ - # ( - # # Case 1: Tasks are defined inside a module without using explicit Aiotaskq - # # instance, and - # "sample_apps.simple_app.aiotaskq", - # # We should instantiate Aiotaskq using the module path. - # simple_app, - # ), ( - # Case 2.1: Tasks are defined using an explicit Aiotaskq instance, and + # Case 1.1: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq:app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 2.2: Tasks are defined using an explicit Aiotaskq instance, and + # Case 1.2: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 2.3: Tasks are defined using an explicit Aiotaskq instance, and + # Case 1.3: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), + ( + # Case 2: Tasks are defined inside a module without using explicit Aiotaskq + # instance, and + "sample_apps.simple_app_implicit_instance.app_aiotaskq", + # We should instantiate a new Aiotaskq instance using the module path. + simple_app_implicit_instance, + ), # ( # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default # # file name ("aiotaskq.py"), and @@ -73,4 +74,8 @@ def test_valid_app_import_path(valid_import_path: str, app_expected: ModuleType) # Then the instantiated Aiotaskq object should be loaded with the tasks assert isinstance(app_actual.add, Task) - assert app_expected.add == app_actual.task_map["add"] == app_actual.add # type: ignore + assert app_actual.add == app_actual.task_map["add"] == app_expected.add # type: ignore + assert app_actual.add(ls=[2, 3]) == 5 + assert isinstance(app_actual.times, Task) + assert app_actual.times == app_actual.task_map["times"] == app_expected.times # type: ignore + assert app_actual.times(x=2, y=3) == 6 diff --git a/src/sample_apps/__init__.py b/src/sample_apps/__init__.py index 7c46f98..27b75b0 100644 --- a/src/sample_apps/__init__.py +++ b/src/sample_apps/__init__.py @@ -1,5 +1,6 @@ -from . import simple_app +from . import simple_app, simple_app_implicit_instance __all__ = [ "simple_app", + "simple_app_implicit_instance", ] diff --git a/src/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml index 403951d..3107a56 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -16,12 +16,6 @@ name = "sample_apps" version = "0.0.0" readme = "README.md" description = "A collection of sample apps to test aiotaskq against Celery" -authors = [ - {name = "Imran Ariffin", email = "ariffin.imran@gmail.com"}, -] -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", -] -license = { file = "LICENSE" } + +[tool.setuptools] +packages = ["simple_app", "simple_app_implicit_instance"] diff --git a/src/sample_apps/simple_app_implicit_instance/__init__.py b/src/sample_apps/simple_app_implicit_instance/__init__.py new file mode 100644 index 0000000..4154c42 --- /dev/null +++ b/src/sample_apps/simple_app_implicit_instance/__init__.py @@ -0,0 +1,3 @@ +from . import app_aiotaskq + +__all__ = ["app_aiotaskq"] diff --git a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py new file mode 100644 index 0000000..6746916 --- /dev/null +++ b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py @@ -0,0 +1,51 @@ +import asyncio +import logging +import time + +from aiotaskq.task import task + +logger = logging.getLogger(__name__) + + +@task +def add(ls: list[int]) -> int: + logger.info("add(%s) ...", ls) + time.sleep(5) + ret = sum(x for x in ls) + logger.info("add(%s) -> %s", ls, ret) + return ret + + +@task +def times(x: int, y: int) -> int: + logger.info("times(%s, %s) ...", x, y) + time.sleep(2) + ret = x * y + logger.info("times(%s, %s) -> %s", x, y, ret) + return ret + + +async def get_formula(): + """ + Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). + # TODO (Issue #44): Support chain of tasks + """ + logger.info("get_formula() ...") + x = await times.apply_async(x=1, y=3) + y = await times.apply_async(x=3, y=4) + ret = await add.apply_async([x, y]) + logger.info("get_formula() -> %s", str(ret)) + return ret + + +async def main(): + logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger("simple_app_aiotaskq.app") + logger.info("Simple App (Aiotaskq)") + ret = await get_formula() + logger.info("Result: %s", ret) + assert ret == 15 + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/tests/test_worker.py b/src/tests/test_worker.py index 55d9a90..da46f93 100644 --- a/src/tests/test_worker.py +++ b/src/tests/test_worker.py @@ -56,8 +56,8 @@ def test_incorrect_app(): # Then the worker process should print error message output = str(worker_cli_process_pipe.read()) output_expected = ( - "Error at argument `--app_import_path some.incorrect.app.name`: " - '"some.incorrect.app.name" is not a path to a valid Python module' + "Error at argument `APP`: " + "\"some.incorrect.app.name\" is not a path to a valid Python module" ) assert output_expected in output # And exit immediately with an error exit code @@ -73,8 +73,8 @@ def test_validate_input(): # Then a descriptive error_message should be returned assert error_msg == ( - "Error at argument `--app_import_path some.incorrect.app.name`:" - ' "some.incorrect.app.name" is not a path to a valid Python module' + "Error at argument `APP`: \"some.incorrect.app.name\"" + " is not a path to a valid Python module" ) From ba0050b6e8472becc393a2a2d1ef0836410ba59c Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 20 Nov 2022 19:21:12 -0500 Subject: [PATCH 14/42] Cover all import cases --- src/aiotaskq/app.py | 36 ++++++++++--------- src/aiotaskq/tests/apps/__init__.py | 4 +-- .../apps/simple_app_encapsulated/__init__.py | 6 ---- .../apps/simple_app_encapsulated/aiotaskq.py | 7 ---- .../tests/apps/simple_app_encapsulated/app.py | 8 ----- .../some_module/__init__.py | 0 .../some_module/tasks.py | 6 ---- .../some_other_module/__init__.py | 0 .../some_other_module/some_tasks.py | 0 .../simple_app_encapsulated_2/__init__.py | 6 ---- .../apps/simple_app_encapsulated_2/app.py | 8 ----- .../some_file_name.py | 14 -------- .../some_module/__init__.py | 0 .../some_module/tasks.py | 6 ---- .../some_other_module/__init__.py | 0 .../some_other_module/some_tasks.py | 0 src/aiotaskq/tests/test_app.py | 16 ++++++++- src/sample_apps/simple_app/aiotaskq.py | 4 +++ src/sample_apps/simple_app/tasks_aiotaskq.py | 2 -- src/sample_apps/simple_app/tasks_celery.py | 2 -- .../app_aiotaskq.py | 2 -- test.sh | 6 ++-- 22 files changed, 43 insertions(+), 90 deletions(-) delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/app.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/__init__.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/__init__.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/some_tasks.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/__init__.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/__init__.py delete mode 100644 src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/some_tasks.py diff --git a/src/aiotaskq/app.py b/src/aiotaskq/app.py index b32d064..ecf5f5b 100644 --- a/src/aiotaskq/app.py +++ b/src/aiotaskq/app.py @@ -33,7 +33,6 @@ def __init__( for module_path in include: module = importlib.import_module(module_path) tasks = self._extract_tasks(app_or_module=module) - print(f"module {module}, tasks: {tasks}") self.task_map.update({task_.__name__: task_ for task_ in tasks}) def __getattribute__(self, attr_name: str) -> t.Any: @@ -53,9 +52,9 @@ def from_import_path(cls, app_or_module_path: str) -> "Aiotaskq": app_import_error_msg = f"App or module path `{app_or_module_path}` is not valid." + app_attr_name: t.Optional[str] = None app: "Aiotaskq" if ":" in app_or_module_path: - print("\n0") # Import path points to an Aiotaskq instance -- use it. module_path, app_name = app_or_module_path.split(":") @@ -66,32 +65,38 @@ def from_import_path(cls, app_or_module_path: str) -> "Aiotaskq": app = getattr(module, app_name) else: + app_or_module: "Aiotaskq | ModuleType" + try: app_or_module = importlib.import_module(f"{app_or_module_path}.aiotaskq") except ImportError: try: app_or_module = importlib.import_module(app_or_module_path) + except ModuleNotFoundError: + module_path, _, app_attr_name = app_or_module_path.rpartition(".") + try: + app_or_module = importlib.import_module(module_path) + except (ModuleNotFoundError, ImportError) as exc: + raise AppImportError(app_import_error_msg) from exc + app = getattr(app_or_module, app_attr_name) except ImportError as exc: raise AppImportError(app_import_error_msg) from exc - if isinstance(app_or_module, Aiotaskq): - # Import path points to an Aiotaskq instance -- use it. - print("\n1") - app = app_or_module + if ( + app_or_module is not None + and app_attr_name is not None + and hasattr(app_or_module, app_attr_name) + and isinstance(getattr(app_or_module, app_attr_name), Aiotaskq) + ): + # Import path points to a module that contains an Aiotaskq instance named as + # `` (or the default "app") -- retrieve the instance and use it. + app = getattr(app_or_module, app_attr_name) elif hasattr(app_or_module, "app") and isinstance( getattr(app_or_module, "app"), Aiotaskq ): # Import path points to a module that contains an Aiotaskq instance # named as `app` -- retrieve the instance and use it. - print("\n2") app = getattr(app_or_module, "app") - elif hasattr(app_or_module, "aiotaskq") and isinstance( - getattr(app_or_module, "aiotaskq"), Aiotaskq - ): - # Import path points to a module that contains an Aiotaskq instance - # named as `aiotaskq` -- retrieve the instance and use it. - print("\n3") - app = getattr(app_or_module, "aiotaskq") else: # Import path points to neither an Aiotaskq instance, nor a module # containing an Aiotaskq instance. @@ -102,7 +107,6 @@ def from_import_path(cls, app_or_module_path: str) -> "Aiotaskq": raise AppImportError(app_import_error_msg) # The Aiotaskq instance or module contains some Task -- instantiate a # new Aiotaskq instance and return it. - print(f"\n4: {dict((t.__name__, t) for t in tasks)}") app = cls(tasks=tasks) app.import_path = app_or_module_path @@ -117,7 +121,7 @@ def register_task(self, func: t.Callable[P, RT]) -> Task[P, RT]: return task_ @classmethod - def _extract_tasks(cls, app_or_module: "Aiotaskq | ModuleType") -> list[Task]: + def _extract_tasks(cls, app_or_module: "ModuleType") -> list[Task]: return [ getattr(app_or_module, attr) for attr in app_or_module.__dict__ diff --git a/src/aiotaskq/tests/apps/__init__.py b/src/aiotaskq/tests/apps/__init__.py index 7337e91..30b7eee 100644 --- a/src/aiotaskq/tests/apps/__init__.py +++ b/src/aiotaskq/tests/apps/__init__.py @@ -1,5 +1,3 @@ from . import simple_app -from . import simple_app_encapsulated -from . import simple_app_encapsulated_2 -__all__ = ["simple_app", "simple_app_encapsulated", "simple_app_encapsulated_2"] +__all__ = ["simple_app"] diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py deleted file mode 100644 index abdcab5..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import aiotaskq -from . import app -from . import some_module -from . import some_other_module - -__all__ = ["aiotaskq", "app", "some_module", "some_other_module"] diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py deleted file mode 100644 index 15436ed..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated/aiotaskq.py +++ /dev/null @@ -1,7 +0,0 @@ -from aiotaskq.app import Aiotaskq - -app = Aiotaskq( - # include=[ - # "aiotaskq.tests.apps.simple_app_encapsulated.some_module.tasks", - # ], -) diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/app.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/app.py deleted file mode 100644 index 905a39f..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated/app.py +++ /dev/null @@ -1,8 +0,0 @@ -"""A sample module that contains an app instance that encapsulates all app logic.""" - - -class SomeSimpleApp: - pass - - -some_simple_app = SomeSimpleApp() diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py deleted file mode 100644 index 6557315..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_module/tasks.py +++ /dev/null @@ -1,6 +0,0 @@ -from ..aiotaskq import app - - -@app.register_task -def add(a: int, b: int) -> int: - return a + b diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/some_tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated/some_other_module/some_tasks.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py deleted file mode 100644 index 5fccda5..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import some_file_name -from . import app -from . import some_module -from . import some_other_module - -__all__ = ["app", "some_file_name", "some_module", "some_other_module"] diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py deleted file mode 100644 index 905a39f..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/app.py +++ /dev/null @@ -1,8 +0,0 @@ -"""A sample module that contains an app instance that encapsulates all app logic.""" - - -class SomeSimpleApp: - pass - - -some_simple_app = SomeSimpleApp() diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py deleted file mode 100644 index cd25013..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_file_name.py +++ /dev/null @@ -1,14 +0,0 @@ -import time - -from aiotaskq.app import Aiotaskq - -app = Aiotaskq( - # includes=[".tasks"], -) -some_app = app - - -@app.register_task -def add(ls: list[int]) -> int: - time.sleep(5) - return sum(x for x in ls) diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py deleted file mode 100644 index 2b2f5de..0000000 --- a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_module/tasks.py +++ /dev/null @@ -1,6 +0,0 @@ -from ..some_file_name import some_app - - -@some_app.register_task -def add(a: int, b: int) -> int: - return a + b diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/__init__.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/some_tasks.py b/src/aiotaskq/tests/apps/simple_app_encapsulated_2/some_other_module/some_tasks.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index 0192477..7d30cf7 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -13,11 +13,25 @@ "valid_import_path,app_expected", [ ( - # Case 1.1: Tasks are defined using an explicit Aiotaskq instance, and + # Case 1.1.1: Tasks are defined using an explicit Aiotaskq instance with + # default name "app", and "sample_apps.simple_app.aiotaskq:app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), + ( + # Case 1.1.2: Tasks are defined using an explicit Aiotaskq instance with + # default name "app", and + "sample_apps.simple_app.aiotaskq.app", + # We should instantiate Aiotaskq from the import path to the instance. + simple_app, + ), + ( + # Case 1.1.3: Tasks are defined using an explicit Aiotaskq instance with custom name, and + "sample_apps.simple_app.aiotaskq:some_app", + # We should instantiate Aiotaskq from the import path to the instance. + simple_app, + ), ( # Case 1.2: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq", diff --git a/src/sample_apps/simple_app/aiotaskq.py b/src/sample_apps/simple_app/aiotaskq.py index fddceb4..d7c93b5 100644 --- a/src/sample_apps/simple_app/aiotaskq.py +++ b/src/sample_apps/simple_app/aiotaskq.py @@ -4,3 +4,7 @@ app = Aiotaskq( include=["sample_apps.simple_app.tasks_aiotaskq"], ) +# OR can use any variable name for the app instance +some_app = app +# Including `aiotaskq` +aiotaskq = app diff --git a/src/sample_apps/simple_app/tasks_aiotaskq.py b/src/sample_apps/simple_app/tasks_aiotaskq.py index 0948ea4..60ef2e2 100644 --- a/src/sample_apps/simple_app/tasks_aiotaskq.py +++ b/src/sample_apps/simple_app/tasks_aiotaskq.py @@ -10,7 +10,6 @@ @task def add(ls: list[int]) -> int: logger.info("add(%s) ...", ls) - time.sleep(5) ret = sum(x for x in ls) logger.info("add(%s) -> %s", ls, ret) return ret @@ -19,7 +18,6 @@ def add(ls: list[int]) -> int: @task def times(x: int, y: int) -> int: logger.info("times(%s, %s) ...", x, y) - time.sleep(2) ret = x * y logger.info("times(%s, %s) -> %s", x, y, ret) return ret diff --git a/src/sample_apps/simple_app/tasks_celery.py b/src/sample_apps/simple_app/tasks_celery.py index 3fb6faa..28b70b5 100644 --- a/src/sample_apps/simple_app/tasks_celery.py +++ b/src/sample_apps/simple_app/tasks_celery.py @@ -12,7 +12,6 @@ @app.task() def add(ls: list[int]) -> int: logger.info("add(%s) ...", ls) - time.sleep(5) ret = sum(x for x in ls) logger.info("add(%s) -> %s", ls, ret) return ret @@ -21,7 +20,6 @@ def add(ls: list[int]) -> int: @app.task() def times(x: int, y: int) -> int: logger.info("times(%s, %s) ...", x, y) - time.sleep(2) ret = x * y logger.info("times(%s, %s) -> %s", x, y, ret) return ret diff --git a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py index 6746916..9baa25f 100644 --- a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py +++ b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py @@ -10,7 +10,6 @@ @task def add(ls: list[int]) -> int: logger.info("add(%s) ...", ls) - time.sleep(5) ret = sum(x for x in ls) logger.info("add(%s) -> %s", ls, ret) return ret @@ -19,7 +18,6 @@ def add(ls: list[int]) -> int: @task def times(x: int, y: int) -> int: logger.info("times(%s, %s) ...", x, y) - time.sleep(2) ret = x * y logger.info("times(%s, %s) -> %s", x, y, ret) return ret diff --git a/test.sh b/test.sh index 7d14973..e275305 100755 --- a/test.sh +++ b/test.sh @@ -4,9 +4,9 @@ python -m pip install --quiet --upgrade pip echo "Install dependencies" pip install --quiet -e .[dev] -echo "Install sample apps" -PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install --quiet -e file://$PWD/src/sample_apps/ +# echo "Install sample apps" +# PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +# pip install --quiet -e file://$PWD/src/sample_apps/ echo "Erase previous coverage files" coverage erase From 66bca53f41afb7fbd81abc2101933aafa8d489f9 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 20 Nov 2022 22:21:13 -0500 Subject: [PATCH 15/42] WIP: Move sample_apps/ to outside of src/; 1 failed test --- .vscode/settings.json | 4 +- {src/sample_apps => sample_apps}/README.md | 0 .../pyproject.template.toml | 3 -- sample_apps/pyproject.toml | 18 +++++++++ .../src}/sample_apps/__init__.py | 0 .../src}/sample_apps/simple_app/__init__.py | 0 .../src/sample_apps/simple_app/aiotaskq.py | 9 +++++ .../sample_apps/simple_app/app_aiotaskq.py | 9 +++-- .../src}/sample_apps/simple_app/app_celery.py | 0 .../src}/sample_apps/simple_app/celery.py | 0 .../sample_apps/simple_app/tasks_aiotaskq.py | 0 .../sample_apps/simple_app/tasks_celery.py | 0 .../src}/sample_apps/simple_app/tests.py | 0 .../simple_app_implicit_instance/__init__.py | 0 .../app_aiotaskq.py | 7 ++-- src/aiotaskq/tests/test_app.py | 39 +++---------------- src/sample_apps/simple_app/aiotaskq.py | 10 ----- test.sh | 8 ++-- 18 files changed, 48 insertions(+), 59 deletions(-) rename {src/sample_apps => sample_apps}/README.md (100%) rename {src/sample_apps => sample_apps}/pyproject.template.toml (83%) create mode 100644 sample_apps/pyproject.toml rename {src => sample_apps/src}/sample_apps/__init__.py (100%) rename {src => sample_apps/src}/sample_apps/simple_app/__init__.py (100%) create mode 100644 sample_apps/src/sample_apps/simple_app/aiotaskq.py rename {src => sample_apps/src}/sample_apps/simple_app/app_aiotaskq.py (76%) rename {src => sample_apps/src}/sample_apps/simple_app/app_celery.py (100%) rename {src => sample_apps/src}/sample_apps/simple_app/celery.py (100%) rename {src => sample_apps/src}/sample_apps/simple_app/tasks_aiotaskq.py (100%) rename {src => sample_apps/src}/sample_apps/simple_app/tasks_celery.py (100%) rename {src => sample_apps/src}/sample_apps/simple_app/tests.py (100%) rename {src => sample_apps/src}/sample_apps/simple_app_implicit_instance/__init__.py (100%) rename {src => sample_apps/src}/sample_apps/simple_app_implicit_instance/app_aiotaskq.py (90%) delete mode 100644 src/sample_apps/simple_app/aiotaskq.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 897c5b9..01fb82b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { "files.exclude": { "**/.git": true, - "**/.venv": true, + "**/.venv*": true, "**/.hg": true, "**/.DS_Store": true, "**/*.egg-info": true, @@ -12,7 +12,7 @@ "python.testing.unittestEnabled": false, "python.analysis.extraPaths": [ "src/tests/", - "./src/sample_apps" + "./sample_apps/" ], "python.defaultInterpreterPath": "../.venv/bin/python3" } diff --git a/src/sample_apps/README.md b/sample_apps/README.md similarity index 100% rename from src/sample_apps/README.md rename to sample_apps/README.md diff --git a/src/sample_apps/pyproject.template.toml b/sample_apps/pyproject.template.toml similarity index 83% rename from src/sample_apps/pyproject.template.toml rename to sample_apps/pyproject.template.toml index 3107a56..ef81b4f 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/sample_apps/pyproject.template.toml @@ -16,6 +16,3 @@ name = "sample_apps" version = "0.0.0" readme = "README.md" description = "A collection of sample apps to test aiotaskq against Celery" - -[tool.setuptools] -packages = ["simple_app", "simple_app_implicit_instance"] diff --git a/sample_apps/pyproject.toml b/sample_apps/pyproject.toml new file mode 100644 index 0000000..6621eb6 --- /dev/null +++ b/sample_apps/pyproject.toml @@ -0,0 +1,18 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" + +[project] +requires-python = ">=3.9" +dependencies = [ + "aiotaskq @ file:///home/in-gote/workspace/aiotaskq/", + "celery >= 5.2.0, < 5.3.0", + "redis >= 4.3.4, < 4.4.0", +] +name = "sample_apps" +version = "0.0.0" +readme = "README.md" +description = "A collection of sample apps to test aiotaskq against Celery" diff --git a/src/sample_apps/__init__.py b/sample_apps/src/sample_apps/__init__.py similarity index 100% rename from src/sample_apps/__init__.py rename to sample_apps/src/sample_apps/__init__.py diff --git a/src/sample_apps/simple_app/__init__.py b/sample_apps/src/sample_apps/simple_app/__init__.py similarity index 100% rename from src/sample_apps/simple_app/__init__.py rename to sample_apps/src/sample_apps/simple_app/__init__.py diff --git a/sample_apps/src/sample_apps/simple_app/aiotaskq.py b/sample_apps/src/sample_apps/simple_app/aiotaskq.py new file mode 100644 index 0000000..8aef022 --- /dev/null +++ b/sample_apps/src/sample_apps/simple_app/aiotaskq.py @@ -0,0 +1,9 @@ +from aiotaskq import Aiotaskq + +# some_app = Aiotaskq( +# include=["sample_apps.simple_app.tasks_aiotaskq"], +# ) +# # OR can use any variable name for the app instance +# app = some_app +# # Including `aiotaskq` +# aiotaskq = app diff --git a/src/sample_apps/simple_app/app_aiotaskq.py b/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py similarity index 76% rename from src/sample_apps/simple_app/app_aiotaskq.py rename to sample_apps/src/sample_apps/simple_app/app_aiotaskq.py index a016dd8..6f278c9 100644 --- a/src/sample_apps/simple_app/app_aiotaskq.py +++ b/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py @@ -3,6 +3,7 @@ from .tasks_aiotaskq import add, times +logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -12,16 +13,16 @@ async def get_formula(): # TODO (Issue #44): Support chain of tasks """ logger.info("get_formula() ...") - x = await times.apply_async(x=1, y=3) - y = await times.apply_async(x=3, y=4) + x, y = await asyncio.gather( + times.apply_async(x=1, y=3), + times.apply_async(x=3, y=4), + ) ret = await add.apply_async([x, y]) logger.info("get_formula() -> %s", str(ret)) return ret async def main(): - logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger("simple_app_aiotaskq.app") logger.info("Simple App (Aiotaskq)") ret = await get_formula() logger.info("Result: %s", ret) diff --git a/src/sample_apps/simple_app/app_celery.py b/sample_apps/src/sample_apps/simple_app/app_celery.py similarity index 100% rename from src/sample_apps/simple_app/app_celery.py rename to sample_apps/src/sample_apps/simple_app/app_celery.py diff --git a/src/sample_apps/simple_app/celery.py b/sample_apps/src/sample_apps/simple_app/celery.py similarity index 100% rename from src/sample_apps/simple_app/celery.py rename to sample_apps/src/sample_apps/simple_app/celery.py diff --git a/src/sample_apps/simple_app/tasks_aiotaskq.py b/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py similarity index 100% rename from src/sample_apps/simple_app/tasks_aiotaskq.py rename to sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py diff --git a/src/sample_apps/simple_app/tasks_celery.py b/sample_apps/src/sample_apps/simple_app/tasks_celery.py similarity index 100% rename from src/sample_apps/simple_app/tasks_celery.py rename to sample_apps/src/sample_apps/simple_app/tasks_celery.py diff --git a/src/sample_apps/simple_app/tests.py b/sample_apps/src/sample_apps/simple_app/tests.py similarity index 100% rename from src/sample_apps/simple_app/tests.py rename to sample_apps/src/sample_apps/simple_app/tests.py diff --git a/src/sample_apps/simple_app_implicit_instance/__init__.py b/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py similarity index 100% rename from src/sample_apps/simple_app_implicit_instance/__init__.py rename to sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py diff --git a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py similarity index 90% rename from src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py rename to sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py index 9baa25f..f8e4020 100644 --- a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py +++ b/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py @@ -1,6 +1,5 @@ import asyncio import logging -import time from aiotaskq.task import task @@ -29,8 +28,10 @@ async def get_formula(): # TODO (Issue #44): Support chain of tasks """ logger.info("get_formula() ...") - x = await times.apply_async(x=1, y=3) - y = await times.apply_async(x=3, y=4) + x, y = await asyncio.gather( + times.apply_async(x=1, y=3), + times.apply_async(x=3, y=4), + ) ret = await add.apply_async([x, y]) logger.info("get_formula() -> %s", str(ret)) return ret diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index 7d30cf7..2294476 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -13,71 +13,44 @@ "valid_import_path,app_expected", [ ( - # Case 1.1.1: Tasks are defined using an explicit Aiotaskq instance with + # Case 1: Tasks are defined using an explicit Aiotaskq instance with # default name "app", and "sample_apps.simple_app.aiotaskq:app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.1.2: Tasks are defined using an explicit Aiotaskq instance with + # Case 2: Tasks are defined using an explicit Aiotaskq instance with # default name "app", and "sample_apps.simple_app.aiotaskq.app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.1.3: Tasks are defined using an explicit Aiotaskq instance with custom name, and + # Case 3: Tasks are defined using an explicit Aiotaskq instance with custom name, and "sample_apps.simple_app.aiotaskq:some_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.2: Tasks are defined using an explicit Aiotaskq instance, and + # Case 4: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.3: Tasks are defined using an explicit Aiotaskq instance, and + # Case 5: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 2: Tasks are defined inside a module without using explicit Aiotaskq + # Case 6: Tasks are defined inside a module without using explicit Aiotaskq # instance, and "sample_apps.simple_app_implicit_instance.app_aiotaskq", # We should instantiate a new Aiotaskq instance using the module path. simple_app_implicit_instance, ), - # ( - # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default - # # file name ("aiotaskq.py"), and - # "aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq:app", - # # We should instantiate Aiotaskq from the import path to the instance - # # using the ":" pattern (this pattern is useful to differentiate between - # # a module and an object). - # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, - # ), - # ( - # # Case 4: Tasks are defined using an explicit Aiotaskq instance in - # # a non-default file name ("some_module_name.py"), and - # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name:some_app", - # # We should instantiate Aiotaskq from the import path to the instance - # # using the ":" pattern (this pattern is useful to differentiate between - # # a module and an object)". - # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, - # ), - # ( - # # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default - # # file name ("some_module_name.py"), and - # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name", - # # We should instantiate Aiotaskq from the import path to the instance - # # using the ":" pattern (this pattern is useful to differentiate between - # # a module and an object)". - # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, - # ), ], ) def test_valid_app_import_path(valid_import_path: str, app_expected: ModuleType): diff --git a/src/sample_apps/simple_app/aiotaskq.py b/src/sample_apps/simple_app/aiotaskq.py deleted file mode 100644 index d7c93b5..0000000 --- a/src/sample_apps/simple_app/aiotaskq.py +++ /dev/null @@ -1,10 +0,0 @@ -from aiotaskq import Aiotaskq - - -app = Aiotaskq( - include=["sample_apps.simple_app.tasks_aiotaskq"], -) -# OR can use any variable name for the app instance -some_app = app -# Including `aiotaskq` -aiotaskq = app diff --git a/test.sh b/test.sh index e275305..df52313 100755 --- a/test.sh +++ b/test.sh @@ -2,11 +2,11 @@ echo "Upgrade pip" python -m pip install --quiet --upgrade pip echo "Install dependencies" -pip install --quiet -e .[dev] +pip install --quiet --no-cache-dir -e .[dev] -# echo "Install sample apps" -# PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -# pip install --quiet -e file://$PWD/src/sample_apps/ +echo "Install sample apps" +PROJECT_DIR=$PWD envsubst < ./sample_apps/pyproject.template.toml > ./sample_apps/pyproject.toml || exit 1 +pip install --quiet --no-cache-dir -e file://$PWD/sample_apps/ || exit 1 echo "Erase previous coverage files" coverage erase From c620840fd6cb752bb1834c911690d7d2ea1ced60 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 20 Nov 2022 22:23:35 -0500 Subject: [PATCH 16/42] Revert "WIP: Move sample_apps/ to outside of src/; 1 failed test" This reverts commit fc2482f24627b76318ad2e484a47edebbb15a6ce. --- .vscode/settings.json | 4 +- sample_apps/pyproject.toml | 18 --------- .../src/sample_apps/simple_app/aiotaskq.py | 9 ----- src/aiotaskq/tests/test_app.py | 39 ++++++++++++++++--- {sample_apps => src/sample_apps}/README.md | 0 .../src => src}/sample_apps/__init__.py | 0 .../sample_apps}/pyproject.template.toml | 3 ++ .../sample_apps/simple_app/__init__.py | 0 src/sample_apps/simple_app/aiotaskq.py | 10 +++++ .../sample_apps/simple_app/app_aiotaskq.py | 9 ++--- .../sample_apps/simple_app/app_celery.py | 0 .../sample_apps/simple_app/celery.py | 0 .../sample_apps/simple_app/tasks_aiotaskq.py | 0 .../sample_apps/simple_app/tasks_celery.py | 0 .../sample_apps/simple_app/tests.py | 0 .../simple_app_implicit_instance/__init__.py | 0 .../app_aiotaskq.py | 7 ++-- test.sh | 8 ++-- 18 files changed, 59 insertions(+), 48 deletions(-) delete mode 100644 sample_apps/pyproject.toml delete mode 100644 sample_apps/src/sample_apps/simple_app/aiotaskq.py rename {sample_apps => src/sample_apps}/README.md (100%) rename {sample_apps/src => src}/sample_apps/__init__.py (100%) rename {sample_apps => src/sample_apps}/pyproject.template.toml (83%) rename {sample_apps/src => src}/sample_apps/simple_app/__init__.py (100%) create mode 100644 src/sample_apps/simple_app/aiotaskq.py rename {sample_apps/src => src}/sample_apps/simple_app/app_aiotaskq.py (76%) rename {sample_apps/src => src}/sample_apps/simple_app/app_celery.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/celery.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/tasks_aiotaskq.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/tasks_celery.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app/tests.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app_implicit_instance/__init__.py (100%) rename {sample_apps/src => src}/sample_apps/simple_app_implicit_instance/app_aiotaskq.py (90%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 01fb82b..897c5b9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { "files.exclude": { "**/.git": true, - "**/.venv*": true, + "**/.venv": true, "**/.hg": true, "**/.DS_Store": true, "**/*.egg-info": true, @@ -12,7 +12,7 @@ "python.testing.unittestEnabled": false, "python.analysis.extraPaths": [ "src/tests/", - "./sample_apps/" + "./src/sample_apps" ], "python.defaultInterpreterPath": "../.venv/bin/python3" } diff --git a/sample_apps/pyproject.toml b/sample_apps/pyproject.toml deleted file mode 100644 index 6621eb6..0000000 --- a/sample_apps/pyproject.toml +++ /dev/null @@ -1,18 +0,0 @@ -[build-system] -requires = [ - "setuptools>=42", - "wheel" -] -build-backend = "setuptools.build_meta" - -[project] -requires-python = ">=3.9" -dependencies = [ - "aiotaskq @ file:///home/in-gote/workspace/aiotaskq/", - "celery >= 5.2.0, < 5.3.0", - "redis >= 4.3.4, < 4.4.0", -] -name = "sample_apps" -version = "0.0.0" -readme = "README.md" -description = "A collection of sample apps to test aiotaskq against Celery" diff --git a/sample_apps/src/sample_apps/simple_app/aiotaskq.py b/sample_apps/src/sample_apps/simple_app/aiotaskq.py deleted file mode 100644 index 8aef022..0000000 --- a/sample_apps/src/sample_apps/simple_app/aiotaskq.py +++ /dev/null @@ -1,9 +0,0 @@ -from aiotaskq import Aiotaskq - -# some_app = Aiotaskq( -# include=["sample_apps.simple_app.tasks_aiotaskq"], -# ) -# # OR can use any variable name for the app instance -# app = some_app -# # Including `aiotaskq` -# aiotaskq = app diff --git a/src/aiotaskq/tests/test_app.py b/src/aiotaskq/tests/test_app.py index 2294476..7d30cf7 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/aiotaskq/tests/test_app.py @@ -13,44 +13,71 @@ "valid_import_path,app_expected", [ ( - # Case 1: Tasks are defined using an explicit Aiotaskq instance with + # Case 1.1.1: Tasks are defined using an explicit Aiotaskq instance with # default name "app", and "sample_apps.simple_app.aiotaskq:app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 2: Tasks are defined using an explicit Aiotaskq instance with + # Case 1.1.2: Tasks are defined using an explicit Aiotaskq instance with # default name "app", and "sample_apps.simple_app.aiotaskq.app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 3: Tasks are defined using an explicit Aiotaskq instance with custom name, and + # Case 1.1.3: Tasks are defined using an explicit Aiotaskq instance with custom name, and "sample_apps.simple_app.aiotaskq:some_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 4: Tasks are defined using an explicit Aiotaskq instance, and + # Case 1.2: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 5: Tasks are defined using an explicit Aiotaskq instance, and + # Case 1.3: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 6: Tasks are defined inside a module without using explicit Aiotaskq + # Case 2: Tasks are defined inside a module without using explicit Aiotaskq # instance, and "sample_apps.simple_app_implicit_instance.app_aiotaskq", # We should instantiate a new Aiotaskq instance using the module path. simple_app_implicit_instance, ), + # ( + # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default + # # file name ("aiotaskq.py"), and + # "aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq:app", + # # We should instantiate Aiotaskq from the import path to the instance + # # using the ":" pattern (this pattern is useful to differentiate between + # # a module and an object). + # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, + # ), + # ( + # # Case 4: Tasks are defined using an explicit Aiotaskq instance in + # # a non-default file name ("some_module_name.py"), and + # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name:some_app", + # # We should instantiate Aiotaskq from the import path to the instance + # # using the ":" pattern (this pattern is useful to differentiate between + # # a module and an object)". + # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + # ), + # ( + # # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default + # # file name ("some_module_name.py"), and + # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name", + # # We should instantiate Aiotaskq from the import path to the instance + # # using the ":" pattern (this pattern is useful to differentiate between + # # a module and an object)". + # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + # ), ], ) def test_valid_app_import_path(valid_import_path: str, app_expected: ModuleType): diff --git a/sample_apps/README.md b/src/sample_apps/README.md similarity index 100% rename from sample_apps/README.md rename to src/sample_apps/README.md diff --git a/sample_apps/src/sample_apps/__init__.py b/src/sample_apps/__init__.py similarity index 100% rename from sample_apps/src/sample_apps/__init__.py rename to src/sample_apps/__init__.py diff --git a/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml similarity index 83% rename from sample_apps/pyproject.template.toml rename to src/sample_apps/pyproject.template.toml index ef81b4f..3107a56 100644 --- a/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -16,3 +16,6 @@ name = "sample_apps" version = "0.0.0" readme = "README.md" description = "A collection of sample apps to test aiotaskq against Celery" + +[tool.setuptools] +packages = ["simple_app", "simple_app_implicit_instance"] diff --git a/sample_apps/src/sample_apps/simple_app/__init__.py b/src/sample_apps/simple_app/__init__.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/__init__.py rename to src/sample_apps/simple_app/__init__.py diff --git a/src/sample_apps/simple_app/aiotaskq.py b/src/sample_apps/simple_app/aiotaskq.py new file mode 100644 index 0000000..d7c93b5 --- /dev/null +++ b/src/sample_apps/simple_app/aiotaskq.py @@ -0,0 +1,10 @@ +from aiotaskq import Aiotaskq + + +app = Aiotaskq( + include=["sample_apps.simple_app.tasks_aiotaskq"], +) +# OR can use any variable name for the app instance +some_app = app +# Including `aiotaskq` +aiotaskq = app diff --git a/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py b/src/sample_apps/simple_app/app_aiotaskq.py similarity index 76% rename from sample_apps/src/sample_apps/simple_app/app_aiotaskq.py rename to src/sample_apps/simple_app/app_aiotaskq.py index 6f278c9..a016dd8 100644 --- a/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py +++ b/src/sample_apps/simple_app/app_aiotaskq.py @@ -3,7 +3,6 @@ from .tasks_aiotaskq import add, times -logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -13,16 +12,16 @@ async def get_formula(): # TODO (Issue #44): Support chain of tasks """ logger.info("get_formula() ...") - x, y = await asyncio.gather( - times.apply_async(x=1, y=3), - times.apply_async(x=3, y=4), - ) + x = await times.apply_async(x=1, y=3) + y = await times.apply_async(x=3, y=4) ret = await add.apply_async([x, y]) logger.info("get_formula() -> %s", str(ret)) return ret async def main(): + logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger("simple_app_aiotaskq.app") logger.info("Simple App (Aiotaskq)") ret = await get_formula() logger.info("Result: %s", ret) diff --git a/sample_apps/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/simple_app/app_celery.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/app_celery.py rename to src/sample_apps/simple_app/app_celery.py diff --git a/sample_apps/src/sample_apps/simple_app/celery.py b/src/sample_apps/simple_app/celery.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/celery.py rename to src/sample_apps/simple_app/celery.py diff --git a/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py b/src/sample_apps/simple_app/tasks_aiotaskq.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py rename to src/sample_apps/simple_app/tasks_aiotaskq.py diff --git a/sample_apps/src/sample_apps/simple_app/tasks_celery.py b/src/sample_apps/simple_app/tasks_celery.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/tasks_celery.py rename to src/sample_apps/simple_app/tasks_celery.py diff --git a/sample_apps/src/sample_apps/simple_app/tests.py b/src/sample_apps/simple_app/tests.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app/tests.py rename to src/sample_apps/simple_app/tests.py diff --git a/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py b/src/sample_apps/simple_app_implicit_instance/__init__.py similarity index 100% rename from sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py rename to src/sample_apps/simple_app_implicit_instance/__init__.py diff --git a/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py similarity index 90% rename from sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py rename to src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py index f8e4020..9baa25f 100644 --- a/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py +++ b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py @@ -1,5 +1,6 @@ import asyncio import logging +import time from aiotaskq.task import task @@ -28,10 +29,8 @@ async def get_formula(): # TODO (Issue #44): Support chain of tasks """ logger.info("get_formula() ...") - x, y = await asyncio.gather( - times.apply_async(x=1, y=3), - times.apply_async(x=3, y=4), - ) + x = await times.apply_async(x=1, y=3) + y = await times.apply_async(x=3, y=4) ret = await add.apply_async([x, y]) logger.info("get_formula() -> %s", str(ret)) return ret diff --git a/test.sh b/test.sh index df52313..e275305 100755 --- a/test.sh +++ b/test.sh @@ -2,11 +2,11 @@ echo "Upgrade pip" python -m pip install --quiet --upgrade pip echo "Install dependencies" -pip install --quiet --no-cache-dir -e .[dev] +pip install --quiet -e .[dev] -echo "Install sample apps" -PROJECT_DIR=$PWD envsubst < ./sample_apps/pyproject.template.toml > ./sample_apps/pyproject.toml || exit 1 -pip install --quiet --no-cache-dir -e file://$PWD/sample_apps/ || exit 1 +# echo "Install sample apps" +# PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +# pip install --quiet -e file://$PWD/src/sample_apps/ echo "Erase previous coverage files" coverage erase From fa477154f371921a62d2d01ed4118c601b484d1e Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 20 Nov 2022 23:27:41 -0500 Subject: [PATCH 17/42] Move tests & sample_apps to src/ [Pass tests] * Ensure that upon `python -m build` only aiotaskq is published --- .vscode/settings.json | 2 +- pyproject.toml | 6 +++++- src/aiotaskq/__init__.py | 4 ---- src/aiotaskq/tests/__init__.py | 3 --- src/aiotaskq/tests/apps/__init__.py | 3 --- src/tests/apps/__init__.py | 3 +++ src/{aiotaskq => }/tests/test_app.py | 12 ++++++------ test.sh | 8 ++++---- 8 files changed, 19 insertions(+), 22 deletions(-) delete mode 100644 src/aiotaskq/tests/__init__.py delete mode 100644 src/aiotaskq/tests/apps/__init__.py rename src/{aiotaskq => }/tests/test_app.py (89%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 897c5b9..a6cce3c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,7 @@ "python.testing.unittestEnabled": false, "python.analysis.extraPaths": [ "src/tests/", - "./src/sample_apps" + "src/sample_apps" ], "python.defaultInterpreterPath": "../.venv/bin/python3" } diff --git a/pyproject.toml b/pyproject.toml index ae0e7ca..af64368 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [ - "setuptools>=42", + "setuptools >= 61.0.0", "wheel" ] build-backend = "setuptools.build_meta" @@ -47,7 +47,11 @@ aiotaskq = "aiotaskq.__main__:cli" [tool.setuptools.packages.find] where = ["src"] +<<<<<<< HEAD include = ["aiotaskq*"] +======= +include = ["aiotaskq*"] +>>>>>>> c95ef23... Move tests & sample_apps to src/ [Pass tests] [tool.black] line-length = 100 diff --git a/src/aiotaskq/__init__.py b/src/aiotaskq/__init__.py index baa8f58..834b687 100644 --- a/src/aiotaskq/__init__.py +++ b/src/aiotaskq/__init__.py @@ -35,7 +35,3 @@ async def main(): __version__ = importlib.metadata.version("aiotaskq") __all__ = ["__version__", "task", "Aiotaskq"] - -# # if os.environ["ENV"] == "test" or TYPE_CHECKING: -# from . import tests -# __all__ += ["tests"] diff --git a/src/aiotaskq/tests/__init__.py b/src/aiotaskq/tests/__init__.py deleted file mode 100644 index 4c1e2f0..0000000 --- a/src/aiotaskq/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from . import apps - -__all__ = ["apps"] diff --git a/src/aiotaskq/tests/apps/__init__.py b/src/aiotaskq/tests/apps/__init__.py deleted file mode 100644 index 30b7eee..0000000 --- a/src/aiotaskq/tests/apps/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from . import simple_app - -__all__ = ["simple_app"] diff --git a/src/tests/apps/__init__.py b/src/tests/apps/__init__.py index e69de29..30b7eee 100644 --- a/src/tests/apps/__init__.py +++ b/src/tests/apps/__init__.py @@ -0,0 +1,3 @@ +from . import simple_app + +__all__ = ["simple_app"] diff --git a/src/aiotaskq/tests/test_app.py b/src/tests/test_app.py similarity index 89% rename from src/aiotaskq/tests/test_app.py rename to src/tests/test_app.py index 7d30cf7..a3aa69d 100644 --- a/src/aiotaskq/tests/test_app.py +++ b/src/tests/test_app.py @@ -54,29 +54,29 @@ # ( # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default # # file name ("aiotaskq.py"), and - # "aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq:app", + # "tests.apps.simple_app_encapsulated.aiotaskq:app", # # We should instantiate Aiotaskq from the import path to the instance # # using the ":" pattern (this pattern is useful to differentiate between # # a module and an object). - # aiotaskq.tests.apps.simple_app_encapsulated.aiotaskq.app, + # tests.apps.simple_app_encapsulated.aiotaskq.app, # ), # ( # # Case 4: Tasks are defined using an explicit Aiotaskq instance in # # a non-default file name ("some_module_name.py"), and - # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name:some_app", + # "tests.apps.simple_app_encapsulated_2.some_file_name:some_app", # # We should instantiate Aiotaskq from the import path to the instance # # using the ":" pattern (this pattern is useful to differentiate between # # a module and an object)". - # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + # tests.apps.simple_app_encapsulated_2.some_file_name.some_app, # ), # ( # # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default # # file name ("some_module_name.py"), and - # "aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name", + # "tests.apps.simple_app_encapsulated_2.some_file_name", # # We should instantiate Aiotaskq from the import path to the instance # # using the ":" pattern (this pattern is useful to differentiate between # # a module and an object)". - # aiotaskq.tests.apps.simple_app_encapsulated_2.some_file_name.some_app, + # tests.apps.simple_app_encapsulated_2.some_file_name.some_app, # ), ], ) diff --git a/test.sh b/test.sh index e275305..4374599 100755 --- a/test.sh +++ b/test.sh @@ -2,11 +2,11 @@ echo "Upgrade pip" python -m pip install --quiet --upgrade pip echo "Install dependencies" -pip install --quiet -e .[dev] +pip install --no-cache-dir -e .[dev] -# echo "Install sample apps" -# PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -# pip install --quiet -e file://$PWD/src/sample_apps/ +echo "Install sample apps" +PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +pip install --no-cache-dir -e file://$PWD/src/sample_apps/ echo "Erase previous coverage files" coverage erase From 7274fcd205e4eb2a2ef3cd9bf51ec0dbdc3a6adf Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 20 Nov 2022 23:31:13 -0500 Subject: [PATCH 18/42] Run pylint separately for sample_apps; FAILED --- .pylintrc.sample_apps | 620 ++++++++++++++++++++++++++++++++++++++++++ lint.sh | 9 +- 2 files changed, 626 insertions(+), 3 deletions(-) create mode 100644 .pylintrc.sample_apps diff --git a/.pylintrc.sample_apps b/.pylintrc.sample_apps new file mode 100644 index 0000000..03b833a --- /dev/null +++ b/.pylintrc.sample_apps @@ -0,0 +1,620 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. The default value ignores Emacs file +# locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.10 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + missing-function-docstring, + missing-class-docstring, + missing-module-docstring + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=BaseException, + Exception + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _, + x, + y, + a, + b, + c, + n, + ls + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=110 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no diff --git a/lint.sh b/lint.sh index f135f39..aac843c 100755 --- a/lint.sh +++ b/lint.sh @@ -1,12 +1,15 @@ if [ -z $1 ]; then pylint -v --rcfile ./.pylintrc src/aiotaskq - failed_2=$? + failed_1=$? pylint -v --rcfile ./.pylintrc.tests src/tests/ - failed_1=$? + failed_2=$? + + pylint -v --rcfile ./.pylintrc.sample_apps src/sample_apps/ + failed_3=$? - [ $failed_1 -eq 0 ] && [ $failed_2 -eq 0 ] && failed=0 || failed=1 + [ $failed_1 -eq 0 ] && [ $failed_2 -eq 0 ] && [ $failed_3 -eq 0 ] && failed=0 || failed=1 else pylint -v $1 failed=$? From 0377781ab3e4f19ef241610e4f403c482ecfb375 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 20 Nov 2022 23:34:26 -0500 Subject: [PATCH 19/42] Remove unnecessary test import cases --- src/tests/test_app.py | 39 ++++++--------------------------------- test.sh | 4 ++-- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/src/tests/test_app.py b/src/tests/test_app.py index a3aa69d..2294476 100644 --- a/src/tests/test_app.py +++ b/src/tests/test_app.py @@ -13,71 +13,44 @@ "valid_import_path,app_expected", [ ( - # Case 1.1.1: Tasks are defined using an explicit Aiotaskq instance with + # Case 1: Tasks are defined using an explicit Aiotaskq instance with # default name "app", and "sample_apps.simple_app.aiotaskq:app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.1.2: Tasks are defined using an explicit Aiotaskq instance with + # Case 2: Tasks are defined using an explicit Aiotaskq instance with # default name "app", and "sample_apps.simple_app.aiotaskq.app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.1.3: Tasks are defined using an explicit Aiotaskq instance with custom name, and + # Case 3: Tasks are defined using an explicit Aiotaskq instance with custom name, and "sample_apps.simple_app.aiotaskq:some_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.2: Tasks are defined using an explicit Aiotaskq instance, and + # Case 4: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app.aiotaskq", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 1.3: Tasks are defined using an explicit Aiotaskq instance, and + # Case 5: Tasks are defined using an explicit Aiotaskq instance, and "sample_apps.simple_app", # We should instantiate Aiotaskq from the import path to the instance. simple_app, ), ( - # Case 2: Tasks are defined inside a module without using explicit Aiotaskq + # Case 6: Tasks are defined inside a module without using explicit Aiotaskq # instance, and "sample_apps.simple_app_implicit_instance.app_aiotaskq", # We should instantiate a new Aiotaskq instance using the module path. simple_app_implicit_instance, ), - # ( - # # Case 3: Tasks are defined using an explicit Aiotaskq instance in a default - # # file name ("aiotaskq.py"), and - # "tests.apps.simple_app_encapsulated.aiotaskq:app", - # # We should instantiate Aiotaskq from the import path to the instance - # # using the ":" pattern (this pattern is useful to differentiate between - # # a module and an object). - # tests.apps.simple_app_encapsulated.aiotaskq.app, - # ), - # ( - # # Case 4: Tasks are defined using an explicit Aiotaskq instance in - # # a non-default file name ("some_module_name.py"), and - # "tests.apps.simple_app_encapsulated_2.some_file_name:some_app", - # # We should instantiate Aiotaskq from the import path to the instance - # # using the ":" pattern (this pattern is useful to differentiate between - # # a module and an object)". - # tests.apps.simple_app_encapsulated_2.some_file_name.some_app, - # ), - # ( - # # Case 5: Tasks are defined using an explicit Aiotaskq instance in a non-default - # # file name ("some_module_name.py"), and - # "tests.apps.simple_app_encapsulated_2.some_file_name", - # # We should instantiate Aiotaskq from the import path to the instance - # # using the ":" pattern (this pattern is useful to differentiate between - # # a module and an object)". - # tests.apps.simple_app_encapsulated_2.some_file_name.some_app, - # ), ], ) def test_valid_app_import_path(valid_import_path: str, app_expected: ModuleType): diff --git a/test.sh b/test.sh index 4374599..7d14973 100755 --- a/test.sh +++ b/test.sh @@ -2,11 +2,11 @@ echo "Upgrade pip" python -m pip install --quiet --upgrade pip echo "Install dependencies" -pip install --no-cache-dir -e .[dev] +pip install --quiet -e .[dev] echo "Install sample apps" PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install --no-cache-dir -e file://$PWD/src/sample_apps/ +pip install --quiet -e file://$PWD/src/sample_apps/ echo "Erase previous coverage files" coverage erase From 3901abba3677a108a9a00ceaf14a910234afb7d0 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Mon, 21 Nov 2022 21:05:25 -0500 Subject: [PATCH 20/42] Fixup --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af64368..e7c34c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,11 +47,7 @@ aiotaskq = "aiotaskq.__main__:cli" [tool.setuptools.packages.find] where = ["src"] -<<<<<<< HEAD -include = ["aiotaskq*"] -======= include = ["aiotaskq*"] ->>>>>>> c95ef23... Move tests & sample_apps to src/ [Pass tests] [tool.black] line-length = 100 From 7d1066feb644a35285da634e0848e7a5585d8d0f Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Mon, 21 Nov 2022 23:01:22 -0500 Subject: [PATCH 21/42] Add test to ensure parity with Celery: All tests PASSED --- src/sample_apps/simple_app/app_aiotaskq.py | 1 + src/sample_apps/simple_app/app_celery.py | 1 + src/sample_apps/simple_app/celery.py | 3 - src/tests/conftest.py | 96 +++++++++++++------ src/tests/test_celery_parity.py | 46 +++++++++ ...tegration.py => test_sync_async_parity.py} | 4 +- src/tests/test_worker.py | 12 +-- 7 files changed, 125 insertions(+), 38 deletions(-) create mode 100644 src/tests/test_celery_parity.py rename src/tests/{test_integration.py => test_sync_async_parity.py} (92%) diff --git a/src/sample_apps/simple_app/app_aiotaskq.py b/src/sample_apps/simple_app/app_aiotaskq.py index a016dd8..1b2e252 100644 --- a/src/sample_apps/simple_app/app_aiotaskq.py +++ b/src/sample_apps/simple_app/app_aiotaskq.py @@ -26,6 +26,7 @@ async def main(): ret = await get_formula() logger.info("Result: %s", ret) assert ret == 15 + return ret if __name__ == "__main__": diff --git a/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/simple_app/app_celery.py index 5115253..7bb64da 100644 --- a/src/sample_apps/simple_app/app_celery.py +++ b/src/sample_apps/simple_app/app_celery.py @@ -27,6 +27,7 @@ def main(): ret = get_formula().apply_async().get() logger.info("Result: %s", ret) assert ret == 15 + return ret if __name__ == "__main__": diff --git a/src/sample_apps/simple_app/celery.py b/src/sample_apps/simple_app/celery.py index d40ddf4..ca093f6 100644 --- a/src/sample_apps/simple_app/celery.py +++ b/src/sample_apps/simple_app/celery.py @@ -6,6 +6,3 @@ ) app.conf.broker_url = "redis://localhost:6379/0" app.conf.result_backend = "redis://localhost:6379/0" - -if __name__ == "__main__": - app.start() diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 77d7a9a..bc26f7b 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -1,4 +1,6 @@ import asyncio +from abc import ABC, abstractmethod +import importlib import multiprocessing import os import signal @@ -9,17 +11,57 @@ from aiotaskq.interfaces import ConcurrencyType from aiotaskq.worker import Defaults, run_worker_forever +if t.TYPE_CHECKING: # pragma: no cover + from celery import Celery -class WorkerFixture: + +class _WorkerFixtureBase(ABC): + proc: multiprocessing.Process + + @abstractmethod + async def start(self, *args, **kwargs) -> None: + """Start worker process.""" + + def terminate(self): + """Send TERM signal to the worker process, and wait for it to exit.""" + if self.proc.is_alive(): + self.proc.terminate() + self.proc.join(timeout=5) + + def interrupt(self) -> None: + """Send INT signal to the worker process, and wait for it to exit.""" + if self.proc.is_alive(): + os.kill(self.proc.pid, signal.SIGINT) # type: ignore + self.proc.join(timeout=5) + + def close(self) -> None: + """Release all resources belonging to the worker process.""" + self.proc.close() + + async def _start_and_set_proc(self, proc: "multiprocessing.Process") -> None: + proc.start() + # Wait for worker to be ready, otherwise some tests will get stuck, because + # we're publishing a task before the worker managed to suscribe. You can + # replicate this by adding `await asyncio.sleep(1)` right before the line in + # in worker.py where the worker manager calls `await pubsub.subscribe()`. + await asyncio.sleep(1.0) + self.proc = proc + + +class WorkerFixtureAiotaskq(_WorkerFixtureBase): proc: multiprocessing.Process + @property + def pid(self) -> t.Optional[int]: + return self.proc.pid if self.proc is not None else None + async def start( self, app: str, concurrency: t.Optional[int] = Defaults.concurrency, concurrency_type: t.Optional[ConcurrencyType] = Defaults.concurrency_type, poll_interval_s: t.Optional[float] = Defaults.poll_interval_s, - ) -> None: + ): proc = multiprocessing.Process( target=lambda: run_worker_forever( app_import_path=app, @@ -28,38 +70,38 @@ async def start( poll_interval_s=poll_interval_s, ) ) - proc.start() - # Wait for worker to be ready, otherwise some tests will get stuck, because - # we're publishing a task before the worker managed to suscribe. You can - # replicate this by adding `await asyncio.sleep(1)` right before the line in - # in worker.py where the worker manager calls `await pubsub.subscribe()`. - await asyncio.sleep(1.0) - self.proc = proc - - @property - def pid(self) -> t.Optional[int]: - return self.proc.pid if self.proc is not None else None + await self._start_and_set_proc(proc=proc) - def terminate(self): - """Send TERM signal to the worker process, and wait for it to exit.""" - if self.proc.is_alive(): - self.proc.terminate() - self.proc.join(timeout=5) - def interrupt(self) -> None: - """Send INT signal to the worker process, and wait for it to exit.""" - if self.proc.is_alive(): - os.kill(self.proc.pid, signal.SIGINT) # type: ignore - self.proc.join(timeout=5) +class WorkerFixtureCelery(_WorkerFixtureBase): + proc: multiprocessing.Process - def close(self) -> None: - """Release all resources belonging to the worker process.""" - self.proc.close() + async def start(self, app: str, concurrency: int) -> None: + module_path, app_attr_name = app.split(":") + module = importlib.import_module(module_path) + app_instance: "Celery" = getattr(module, app_attr_name) + proc = multiprocessing.Process( + target=lambda: app_instance.worker_main( + argv=[ + "worker", + f"--concurrency={concurrency}", + ], + ), + ) + await self._start_and_set_proc(proc=proc) @pytest.fixture def worker(): - worker_ = WorkerFixture() + worker_ = WorkerFixtureAiotaskq() + yield worker_ + worker_.terminate() + worker_.close() + + +@pytest.fixture +def worker_celery(): + worker_ = WorkerFixtureCelery() yield worker_ worker_.terminate() worker_.close() diff --git a/src/tests/test_celery_parity.py b/src/tests/test_celery_parity.py new file mode 100644 index 0000000..4b9a7dc --- /dev/null +++ b/src/tests/test_celery_parity.py @@ -0,0 +1,46 @@ +""" +Ensure that aiotaskq has parity with selected celery features. + +The goal is to have an interface as close as possible to Celery +to enable easy adoption. +""" + +import typing as t +from unittest import result + +import pytest + +from sample_apps.simple_app import app_aiotaskq +from sample_apps.simple_app import app_celery + +if t.TYPE_CHECKING: # pragma: no cover + from tests.conftest import WorkerFixtureAiotaskq, WorkerFixtureCelery + + +@pytest.mark.asyncio +async def test_parity_with_celery__simple_app(worker: "WorkerFixtureAiotaskq", worker_celery: "WorkerFixtureCelery"): + # Given a simple app that uses both Celery and aiotaskq in + # the following file structure (Please check it out): + # > tree src/sample_apps/simple_app -I __pycache__ + # src/sample_apps/simple_app + # ├── aiotaskq.py + # ├── app_aiotaskq.py + # ├── app_celery.py + # ├── celery.py + # ├── __init__.py + # ├── tasks_aiotaskq.py + # ├── tasks_celery.py + # └── tests.py + + concurrency = 1 + # And the simple app is running as Celery worker in background + await worker_celery.start(app="sample_apps.simple_app.celery:app", concurrency=concurrency) + # And the simple app is also running as aiotaskq worker in background + await worker.start(app="sample_apps.simple_app.aiotaskq:app", concurrency=concurrency) + + # When both apps are run + result_aiotaskq = await app_aiotaskq.main() + result_celery = app_celery.main() + + # Then they should output the same result + assert result_aiotaskq == result_celery diff --git a/src/tests/test_integration.py b/src/tests/test_sync_async_parity.py similarity index 92% rename from src/tests/test_integration.py rename to src/tests/test_sync_async_parity.py index 31ad89d..b872378 100644 --- a/src/tests/test_integration.py +++ b/src/tests/test_sync_async_parity.py @@ -1,12 +1,12 @@ import pytest from aiotaskq.task import Task -from tests.conftest import WorkerFixture +from tests.conftest import WorkerFixtureAiotaskq from tests.apps import simple_app @pytest.mark.asyncio -async def test_sync_and_async_parity__simple_app(worker: WorkerFixture): +async def test_sync_and_async_parity__simple_app(worker: WorkerFixtureAiotaskq): # Given a simple app running as a worker app = simple_app await worker.start(app=app.__name__, concurrency=8) diff --git a/src/tests/test_worker.py b/src/tests/test_worker.py index da46f93..869eafe 100644 --- a/src/tests/test_worker.py +++ b/src/tests/test_worker.py @@ -9,11 +9,11 @@ from aiotaskq.worker import validate_input if TYPE_CHECKING: # pragma: no cover - from tests.conftest import WorkerFixture + from tests.conftest import WorkerFixtureAiotaskq @pytest.mark.asyncio -async def test_concurrency_starts_child_workers(worker: "WorkerFixture"): +async def test_concurrency_starts_child_workers(worker: "WorkerFixtureAiotaskq"): """ Assert that when --concurrency N option is provided, N child processes will be spawn. """ @@ -29,7 +29,7 @@ async def test_concurrency_starts_child_workers(worker: "WorkerFixture"): @pytest.mark.asyncio async def test_concurrency_starts_child_workers_with_default_concurrency( - worker: "WorkerFixture", + worker: "WorkerFixtureAiotaskq", ): """ Assert that when --concurrency is NOT provided, N child processes will be spawned, N=cpu cores. @@ -79,7 +79,7 @@ def test_validate_input(): @pytest.mark.asyncio -async def test_run_worker__incorrect_app_name(worker: "WorkerFixture"): +async def test_run_worker__incorrect_app_name(worker: "WorkerFixtureAiotaskq"): # Given a worker being started with an incorrect app path await worker.start( app="some.incorrect.app.name", @@ -93,7 +93,7 @@ async def test_run_worker__incorrect_app_name(worker: "WorkerFixture"): @pytest.mark.asyncio -async def test_handle_keyboard_interrupt(worker: "WorkerFixture"): +async def test_handle_keyboard_interrupt(worker: "WorkerFixtureAiotaskq"): # Given a running worker with some child processes concurrency = 4 await worker.start("tests.apps.simple_app", concurrency=concurrency) @@ -115,7 +115,7 @@ async def test_handle_keyboard_interrupt(worker: "WorkerFixture"): @pytest.mark.asyncio -async def test_handle_termination_signal(worker: "WorkerFixture"): +async def test_handle_termination_signal(worker: "WorkerFixtureAiotaskq"): # Given a running worker with some child processes concurrency = 4 await worker.start("tests.apps.simple_app", concurrency=concurrency) From c286786167807bd24ec79f2faee4ce09e425dc4b Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 26 Nov 2022 21:46:52 -0500 Subject: [PATCH 22/42] Cleanup --- .pylintrc.sample_apps | 3 +- src/sample_apps/simple_app/app_aiotaskq.py | 9 ++-- src/sample_apps/simple_app/app_celery.py | 9 ++-- src/sample_apps/simple_app/tasks_aiotaskq.py | 2 - src/sample_apps/simple_app/tasks_celery.py | 4 -- .../simple_app_implicit_instance/__init__.py | 3 +- .../app_aiotaskq.py | 16 +++--- .../app_celery.py | 49 +++++++++++++++++++ .../simple_app_implicit_instance/celery.py | 5 ++ src/tests/conftest.py | 41 +++++++++++----- src/tests/test_app.py | 4 +- src/tests/test_celery_parity.py | 45 +++++++++++++++-- src/tests/test_worker.py | 4 +- 13 files changed, 145 insertions(+), 49 deletions(-) create mode 100644 src/sample_apps/simple_app_implicit_instance/app_celery.py create mode 100644 src/sample_apps/simple_app_implicit_instance/celery.py diff --git a/.pylintrc.sample_apps b/.pylintrc.sample_apps index 03b833a..d8227bf 100644 --- a/.pylintrc.sample_apps +++ b/.pylintrc.sample_apps @@ -154,7 +154,8 @@ disable=raw-checker-failed, use-symbolic-message-instead, missing-function-docstring, missing-class-docstring, - missing-module-docstring + missing-module-docstring, + duplicate-code # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/src/sample_apps/simple_app/app_aiotaskq.py b/src/sample_apps/simple_app/app_aiotaskq.py index 1b2e252..f3c9ced 100644 --- a/src/sample_apps/simple_app/app_aiotaskq.py +++ b/src/sample_apps/simple_app/app_aiotaskq.py @@ -6,24 +6,23 @@ logger = logging.getLogger(__name__) -async def get_formula(): +async def apply_formula(): """ Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). # TODO (Issue #44): Support chain of tasks """ - logger.info("get_formula() ...") + logger.info("apply_formula() ...") x = await times.apply_async(x=1, y=3) y = await times.apply_async(x=3, y=4) ret = await add.apply_async([x, y]) - logger.info("get_formula() -> %s", str(ret)) + logger.info("apply_formula() -> %s", str(ret)) return ret async def main(): logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger("simple_app_aiotaskq.app") logger.info("Simple App (Aiotaskq)") - ret = await get_formula() + ret = await apply_formula() logger.info("Result: %s", ret) assert ret == 15 return ret diff --git a/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/simple_app/app_celery.py index 7bb64da..1dcc5b0 100644 --- a/src/sample_apps/simple_app/app_celery.py +++ b/src/sample_apps/simple_app/app_celery.py @@ -6,25 +6,24 @@ logger = logging.getLogger(__name__) -def get_formula() -> Signature: +def apply_formula() -> Signature: """ Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). """ - logger.info("get_formula() ...") + logger.info("apply_formula() ...") times_tasks: list[Signature] = [ times.si(x=1, y=3), times.si(x=3, y=4), ] ret: Signature = chord(header=group(times_tasks), body=add.s()) - logger.info("get_formula() -> %s", str(ret)) + logger.info("apply_formula() -> %s", str(ret)) return ret def main(): logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger("simple_app_celery.app") logger.info("Simple App (Celery)") - ret = get_formula().apply_async().get() + ret = apply_formula().apply_async().get() logger.info("Result: %s", ret) assert ret == 15 return ret diff --git a/src/sample_apps/simple_app/tasks_aiotaskq.py b/src/sample_apps/simple_app/tasks_aiotaskq.py index 60ef2e2..4f5d4e8 100644 --- a/src/sample_apps/simple_app/tasks_aiotaskq.py +++ b/src/sample_apps/simple_app/tasks_aiotaskq.py @@ -1,6 +1,4 @@ -import asyncio import logging -import time from aiotaskq.task import task diff --git a/src/sample_apps/simple_app/tasks_celery.py b/src/sample_apps/simple_app/tasks_celery.py index 28b70b5..83aac87 100644 --- a/src/sample_apps/simple_app/tasks_celery.py +++ b/src/sample_apps/simple_app/tasks_celery.py @@ -1,8 +1,4 @@ import logging -import time - -from celery.canvas import chord, group -import celery from .celery import app diff --git a/src/sample_apps/simple_app_implicit_instance/__init__.py b/src/sample_apps/simple_app_implicit_instance/__init__.py index 4154c42..b41c2ac 100644 --- a/src/sample_apps/simple_app_implicit_instance/__init__.py +++ b/src/sample_apps/simple_app_implicit_instance/__init__.py @@ -1,3 +1,4 @@ from . import app_aiotaskq +from . import celery -__all__ = ["app_aiotaskq"] +__all__ = ["app_aiotaskq", "celery"] diff --git a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py index 9baa25f..357c8c6 100644 --- a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py +++ b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py @@ -1,6 +1,5 @@ import asyncio import logging -import time from aiotaskq.task import task @@ -23,27 +22,26 @@ def times(x: int, y: int) -> int: return ret -async def get_formula(): +async def apply_formula(): """ Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). # TODO (Issue #44): Support chain of tasks """ - logger.info("get_formula() ...") - x = await times.apply_async(x=1, y=3) - y = await times.apply_async(x=3, y=4) + logger.info("apply_formula() ...") + x, y = await asyncio.gather(times.apply_async(x=1, y=3), times.apply_async(x=3, y=4)) ret = await add.apply_async([x, y]) - logger.info("get_formula() -> %s", str(ret)) + logger.info("apply_formula() -> %s", str(ret)) return ret async def main(): logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger("simple_app_aiotaskq.app") logger.info("Simple App (Aiotaskq)") - ret = await get_formula() + ret = await apply_formula() logger.info("Result: %s", ret) assert ret == 15 + return ret -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover asyncio.run(main()) diff --git a/src/sample_apps/simple_app_implicit_instance/app_celery.py b/src/sample_apps/simple_app_implicit_instance/app_celery.py new file mode 100644 index 0000000..5c24a68 --- /dev/null +++ b/src/sample_apps/simple_app_implicit_instance/app_celery.py @@ -0,0 +1,49 @@ +import logging + +from celery import shared_task +from celery.canvas import Signature, chord, group + +logger = logging.getLogger(__name__) + + +@shared_task +def add(ls: list[int]) -> int: + logger.info("add(%s) ...", ls) + ret = sum(x for x in ls) + logger.info("add(%s) -> %s", ls, ret) + return ret + + +@shared_task +def times(x: int, y: int) -> int: + logger.info("times(%s, %s) ...", x, y) + ret = x * y + logger.info("times(%s, %s) -> %s", x, y, ret) + return ret + + +def apply_formula() -> Signature: + """ + Implement f(x) -> sum(x * y for x, y in zip([1, 2], [3, 4])). + """ + logger.info("apply_formula() ...") + times_tasks: list[Signature] = [ + times.si(x=1, y=3), + times.si(x=3, y=4), + ] + ret: Signature = chord(header=group(times_tasks), body=add.s()) + logger.info("apply_formula() -> %s", str(ret)) + return ret + + +def main(): + logging.basicConfig(level=logging.DEBUG) + logger.info("Simple App (Celery)") + ret = apply_formula().apply_async().get() + logger.info("Result: %s", ret) + assert ret == 15 + return ret + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/src/sample_apps/simple_app_implicit_instance/celery.py b/src/sample_apps/simple_app_implicit_instance/celery.py new file mode 100644 index 0000000..b0b35bd --- /dev/null +++ b/src/sample_apps/simple_app_implicit_instance/celery.py @@ -0,0 +1,5 @@ +from celery import Celery + +app = Celery() +app.conf.broker_url = "redis://localhost:6379/0" +app.conf.result_backend = "redis://localhost:6379/0" diff --git a/src/tests/conftest.py b/src/tests/conftest.py index bc26f7b..0dc5bb4 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -19,9 +19,9 @@ class _WorkerFixtureBase(ABC): proc: multiprocessing.Process @abstractmethod - async def start(self, *args, **kwargs) -> None: + async def start(self, app:str, **kwargs) -> None: """Start worker process.""" - + def terminate(self): """Send TERM signal to the worker process, and wait for it to exit.""" if self.proc.is_alive(): @@ -37,14 +37,14 @@ def interrupt(self) -> None: def close(self) -> None: """Release all resources belonging to the worker process.""" self.proc.close() - + async def _start_and_set_proc(self, proc: "multiprocessing.Process") -> None: proc.start() # Wait for worker to be ready, otherwise some tests will get stuck, because # we're publishing a task before the worker managed to suscribe. You can # replicate this by adding `await asyncio.sleep(1)` right before the line in # in worker.py where the worker manager calls `await pubsub.subscribe()`. - await asyncio.sleep(1.0) + await asyncio.sleep(2.0) self.proc = proc @@ -55,13 +55,10 @@ class WorkerFixtureAiotaskq(_WorkerFixtureBase): def pid(self) -> t.Optional[int]: return self.proc.pid if self.proc is not None else None - async def start( - self, - app: str, - concurrency: t.Optional[int] = Defaults.concurrency, - concurrency_type: t.Optional[ConcurrencyType] = Defaults.concurrency_type, - poll_interval_s: t.Optional[float] = Defaults.poll_interval_s, - ): + async def start(self, app: str, **kwargs): + concurrency: int = kwargs.get("concurrency", Defaults.concurrency) + concurrency_type: ConcurrencyType = kwargs.get("concurrency_type", Defaults.concurrency_type) + poll_interval_s: float = kwargs.get("poll_interval_s", Defaults.poll_interval_s) proc = multiprocessing.Process( target=lambda: run_worker_forever( app_import_path=app, @@ -76,10 +73,17 @@ async def start( class WorkerFixtureCelery(_WorkerFixtureBase): proc: multiprocessing.Process - async def start(self, app: str, concurrency: int) -> None: - module_path, app_attr_name = app.split(":") + async def start(self, app: str, **kwargs) -> None: + concurrency: int = kwargs["concurrency"] + + if ":" in app: + module_path, app_attr_name = app.split(":") + else: + module_path = f"{app}.celery" if not app.endswith(".celery") else app + app_attr_name = "app" module = importlib.import_module(module_path) app_instance: "Celery" = getattr(module, app_attr_name) + proc = multiprocessing.Process( target=lambda: app_instance.worker_main( argv=[ @@ -99,6 +103,17 @@ def worker(): worker_.close() +@pytest.fixture +def worker_aiotaskq(worker: "WorkerFixtureAiotaskq"): # pylint: disable=redefined-outer-name + """ + Define an alias to `worker` fixture. + + This is useful to differentiate between Aiotaskq and Celery worker + in tests where we compare them side by side. + """ + yield worker + + @pytest.fixture def worker_celery(): worker_ = WorkerFixtureCelery() diff --git a/src/tests/test_app.py b/src/tests/test_app.py index 2294476..faac371 100644 --- a/src/tests/test_app.py +++ b/src/tests/test_app.py @@ -61,8 +61,8 @@ def test_valid_app_import_path(valid_import_path: str, app_expected: ModuleType) # Then the instantiated Aiotaskq object should be loaded with the tasks assert isinstance(app_actual.add, Task) - assert app_actual.add == app_actual.task_map["add"] == app_expected.add # type: ignore + assert app_actual.add == app_actual.task_map["add"] == app_expected.add # type: ignore assert app_actual.add(ls=[2, 3]) == 5 assert isinstance(app_actual.times, Task) - assert app_actual.times == app_actual.task_map["times"] == app_expected.times # type: ignore + assert app_actual.times == app_actual.task_map["times"] == app_expected.times # type: ignore assert app_actual.times(x=2, y=3) == 6 diff --git a/src/tests/test_celery_parity.py b/src/tests/test_celery_parity.py index 4b9a7dc..156b5ec 100644 --- a/src/tests/test_celery_parity.py +++ b/src/tests/test_celery_parity.py @@ -6,7 +6,6 @@ """ import typing as t -from unittest import result import pytest @@ -18,8 +17,11 @@ @pytest.mark.asyncio -async def test_parity_with_celery__simple_app(worker: "WorkerFixtureAiotaskq", worker_celery: "WorkerFixtureCelery"): - # Given a simple app that uses both Celery and aiotaskq in +async def test_parity_with_celery__simple_app( + worker_aiotaskq: "WorkerFixtureAiotaskq", + worker_celery: "WorkerFixtureCelery", +): + # Given a simple app that uses both Celery and aiotaskq in # the following file structure (Please check it out): # > tree src/sample_apps/simple_app -I __pycache__ # src/sample_apps/simple_app @@ -32,11 +34,44 @@ async def test_parity_with_celery__simple_app(worker: "WorkerFixtureAiotaskq", w # ├── tasks_celery.py # └── tests.py - concurrency = 1 + concurrency = 2 # And the simple app is running as Celery worker in background await worker_celery.start(app="sample_apps.simple_app.celery:app", concurrency=concurrency) # And the simple app is also running as aiotaskq worker in background - await worker.start(app="sample_apps.simple_app.aiotaskq:app", concurrency=concurrency) + await worker_aiotaskq.start(app="sample_apps.simple_app.aiotaskq:app", concurrency=concurrency) + + # When both apps are run + result_aiotaskq = await app_aiotaskq.main() + result_celery = app_celery.main() + + # Then they should output the same result + assert result_aiotaskq == result_celery + + +@pytest.mark.asyncio +async def test_parity_with_celery__simple_app_implicit_instance( + worker_aiotaskq: "WorkerFixtureAiotaskq", + worker_celery: "WorkerFixtureCelery", +): + # Given a simple app that uses both Celery and aiotaskq in + # the following file structure (Please check it out): + # > tree src/sample_apps/simple_app_implicit -I __pycache__ + # src/sample_apps/simple_app_implicit_instance + # ├── app_aiotaskq.py + # ├── app_celery.py + # └── __init__.py + + concurrency = 2 + # And the simple app is running as Celery worker in background + await worker_celery.start( + app="sample_apps.simple_app_implicit_instance", + concurrency=concurrency, + ) + # And the simple app is also running as aiotaskq worker in background + await worker_aiotaskq.start( + app="sample_apps.simple_app_implicit_instance.app_aiotaskq", + concurrency=concurrency, + ) # When both apps are run result_aiotaskq = await app_aiotaskq.main() diff --git a/src/tests/test_worker.py b/src/tests/test_worker.py index 869eafe..ce83b57 100644 --- a/src/tests/test_worker.py +++ b/src/tests/test_worker.py @@ -57,7 +57,7 @@ def test_incorrect_app(): output = str(worker_cli_process_pipe.read()) output_expected = ( "Error at argument `APP`: " - "\"some.incorrect.app.name\" is not a path to a valid Python module" + '"some.incorrect.app.name" is not a path to a valid Python module' ) assert output_expected in output # And exit immediately with an error exit code @@ -73,7 +73,7 @@ def test_validate_input(): # Then a descriptive error_message should be returned assert error_msg == ( - "Error at argument `APP`: \"some.incorrect.app.name\"" + 'Error at argument `APP`: "some.incorrect.app.name"' " is not a path to a valid Python module" ) From 97cb83ffd3742732e2c06fee8e319657fb338f9a Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Tue, 29 Nov 2022 21:42:33 -0500 Subject: [PATCH 23/42] WIP: Ensure we test sample apps and sync with docs --- .gitignore | 3 + check-docs.sh | 21 +++ check_redis_ready.py | 20 +++ package-lock.json | 149 ++++++++++++++++++ package.json | 14 ++ pyproject.toml | 2 +- src/aiotaskq/app.py | 8 - src/aiotaskq/task.py | 1 - src/aiotaskq/worker.py | 1 - src/sample_apps/README.md | 46 ++++-- src/sample_apps/__init__.py | 6 - .../demo-sample-apps-simple-app-celery.sh | 33 ++++ .../demo-sample-apps-simple-app.sh | 35 ++++ src/sample_apps/pyproject.template.toml | 4 +- src/sample_apps/simple_app/__init__.py | 17 -- src/sample_apps/simple_app/app_aiotaskq.py | 10 +- src/sample_apps/simple_app/app_celery.py | 1 - src/sample_apps/simple_app/tests.py | 0 .../simple_app_implicit_instance/__init__.py | 3 - .../app_aiotaskq.py | 1 - .../app_celery.py | 1 - src/tests/test_docs_sample_apps.py | 21 +++ 22 files changed, 341 insertions(+), 56 deletions(-) create mode 100755 check-docs.sh create mode 100644 check_redis_ready.py create mode 100644 package-lock.json create mode 100644 package.json create mode 100755 src/sample_apps/demo-sample-apps-simple-app-celery.sh create mode 100755 src/sample_apps/demo-sample-apps-simple-app.sh delete mode 100644 src/sample_apps/simple_app/tests.py create mode 100644 src/tests/test_docs_sample_apps.py diff --git a/.gitignore b/.gitignore index 8ecc03e..9540103 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,6 @@ cython_debug/ #.idea/ codecov + +# NPM Scripts +node_modules/ diff --git a/check-docs.sh b/check-docs.sh new file mode 100755 index 0000000..3b1be70 --- /dev/null +++ b/check-docs.sh @@ -0,0 +1,21 @@ +set -ex +npm --version +npm install embedme 2> /dev/null > /dev/null +DOC=src/sample_apps/README.md +npm run update-docs -- $DOC + +set +ex + +echo "Expecting no *uncommitted* changes to $DOC, checking ..." +gitdiff=$(git diff -- $DOC) +if [ -z "$gitdiff" ] +then + echo "Changes to $DOC was already committed. Good." +else + echo "Missing changes to $DOC that are supposed to be commited:" + set -ex + git diff -- $DOC | cat + set +ex + echo "Please run 'npm run -- $DOC', verify the changes, and commit appropriately." + exit 1 +fi \ No newline at end of file diff --git a/check_redis_ready.py b/check_redis_ready.py new file mode 100644 index 0000000..d0600ce --- /dev/null +++ b/check_redis_ready.py @@ -0,0 +1,20 @@ +import asyncio + +import aioredis as redis + + +client: redis.Redis = redis.from_url("redis://127.0.0.1:6379") + +async def _ping_until_ready(): + while True: + try: + response = await client.ping() + if response is True: + print("Redis is ready") + break + except redis.ConnectionError: + wait_s = 0.5 + print(f"Redis is not ready. Try pinging again in {wait_s}") + await asyncio.sleep(wait_s) + +asyncio.run(_ping_until_ready()) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..fbbb86e --- /dev/null +++ b/package-lock.json @@ -0,0 +1,149 @@ +{ + "name": "aiotaskq", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "embedme": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/embedme/-/embedme-1.22.1.tgz", + "integrity": "sha512-wHLuAOI9XoCAQ322mbslIR7PQNgPGYCWrDlYw5C6fesakuhCzi6ce0BrLTZ/EEKgiHEUqcG9V3s7MGO0x1Zgig==", + "requires": { + "chalk": "3.0.0", + "commander": "5.1.0", + "gitignore-parser": "~0.0.2", + "glob": "~7.1.4" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "gitignore-parser": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/gitignore-parser/-/gitignore-parser-0.0.2.tgz", + "integrity": "sha512-X6mpqUv59uWLGD4n3hZ8Cu8KbF2PMWPSFYmxZjdkpm3yOU7hSUYnzTkZI1mcWqchphvqyuz3/BhgBR4E/JtkCg==" + }, + "glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..5971e14 --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "aiotaskq", + "version": "1.0.0", + "main": "npx", + "directories": { + "doc": "docs" + }, + "scripts": { + "update-docs": "embedme" + }, + "dependencies": { + "embedme": "^1.22.1" + } +} diff --git a/pyproject.toml b/pyproject.toml index e7c34c0..590ad47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [ - "setuptools >= 61.0.0", + "setuptools >= 42", "wheel" ] build-backend = "setuptools.build_meta" diff --git a/src/aiotaskq/app.py b/src/aiotaskq/app.py index ecf5f5b..312cfce 100644 --- a/src/aiotaskq/app.py +++ b/src/aiotaskq/app.py @@ -112,14 +112,6 @@ def from_import_path(cls, app_or_module_path: str) -> "Aiotaskq": app.import_path = app_or_module_path return app - def register_task(self, func: t.Callable[P, RT]) -> Task[P, RT]: - """ - Register a function as a task so that it can be retrieved from an Aiotaskq app instance. - """ - task_: Task[P, RT] = task(func) - self.task_map[func.__name__] = task_ - return task_ - @classmethod def _extract_tasks(cls, app_or_module: "ModuleType") -> list[Task]: return [ diff --git a/src/aiotaskq/task.py b/src/aiotaskq/task.py index b084115..8dbb645 100644 --- a/src/aiotaskq/task.py +++ b/src/aiotaskq/task.py @@ -15,7 +15,6 @@ RT = t.TypeVar("RT") P = t.ParamSpec("P") -logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) diff --git a/src/aiotaskq/worker.py b/src/aiotaskq/worker.py index bc6a89f..a700942 100755 --- a/src/aiotaskq/worker.py +++ b/src/aiotaskq/worker.py @@ -18,7 +18,6 @@ from .interfaces import ConcurrencyType, IConcurrencyManager, IPubSub from .pubsub import PubSubSingleton -logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/sample_apps/README.md b/src/sample_apps/README.md index e6ef536..2774bcf 100644 --- a/src/sample_apps/README.md +++ b/src/sample_apps/README.md @@ -28,22 +28,33 @@ Feel free to run this inside of a `aiotaskq` repository. Copy-pasting these commands to your terminal should work out of the box. ```bash +# ./demo-sample-apps-simple-app.sh#L1-L35 + # If not already inside it, clone the aiotaskq repository and cd into it [[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) # Let's say we want to run the first app (Simple App), which is located -# in `./sample_apps/simple_app/`. Update this env var APP as you'd like +# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like # to choose your desired sample app. APP=simple_app # Create a new virtual env specifically for the sample apps -python -m venv ./sample_apps/.venv -source ./sample_apps/.venv/bin/activate +rm -rf ./src/sample_apps/.venv || echo "" +python3.10 -m venv ./src/sample_apps/.venv +source ./src/sample_apps/.venv/bin/activate +echo "Using $(python --version)" # Install sample_apps package from local file -pip install -e sample_apps +python -m pip install --no-cache-dir --upgrade pip +PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +pip install --no-cache-dir file://$PWD/src/sample_apps + +# Start redis and wait for it to be ready +docker-compose up -d redis +python ./check_redis_ready.py # Run aiotaskq workers in background and wait for it be ready +aiotaskq --version aiotaskq worker sample_apps.$APP --concurrency 4 & sleep 2 @@ -53,7 +64,7 @@ python -m sample_apps.$APP.app_aiotaskq # Confirm in the logs if the app is running correctly # Kill aiotaskq workers that were running in background -ps | grep aiotaskq | cut -d ' ' -f 1 | xargs kill -TERM +ps | grep aiotaskq | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM ``` ## Run a sample app with Celery @@ -62,30 +73,39 @@ Now, if you want to run the sample app with `Celery` to see how `aiotaskq` compares to it, do the following: ```bash +# ./demo-sample-apps-simple-app-celery.sh#L1-L33 + # If not already inside it, clone the aiotaskq repository and cd into it [[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) +# Create a new virtual env specifically for the sample apps +python3.10 -m venv ./src/sample_apps/.venv +source ./src/sample_apps/.venv/bin/activate +echo "Using $(python --version)" + +# Start redis and wait for it to be ready +docker-compose up --detach redis +python ./check_redis_ready.py + # Let's say we want to run the first app (Simple App), which is located -# in `./sample_apps/simple_app/`. Update this env var APP as you'd like +# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like # to choose your desired sample app. APP=simple_app -# Create a new virtual env specifically for the sample apps -python -m venv ./sample_apps/.venv -source ./sample_apps/.venv/bin/activate - # Install sample_apps package from local file -pip install -e sample_apps +python3 -m pip install --upgrade pip +PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +pip install -e ./src/sample_apps # Run Celery workers in background and wait for it to be ready celery -A sample_apps.$APP worker --concurrency 4 & sleep 2 # Run the the sample app -python -m sample_apps.$APP.app_celery +python3.10 -m sample_apps.$APP.app_celery # Confirm in the logs if the app is running correctly # Kill celery workers that were running in background -ps | grep celery | cut -d ' ' -f 1 | xargs kill -TERM +ps | grep celery | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM ``` diff --git a/src/sample_apps/__init__.py b/src/sample_apps/__init__.py index 27b75b0..e69de29 100644 --- a/src/sample_apps/__init__.py +++ b/src/sample_apps/__init__.py @@ -1,6 +0,0 @@ -from . import simple_app, simple_app_implicit_instance - -__all__ = [ - "simple_app", - "simple_app_implicit_instance", -] diff --git a/src/sample_apps/demo-sample-apps-simple-app-celery.sh b/src/sample_apps/demo-sample-apps-simple-app-celery.sh new file mode 100755 index 0000000..1c939d4 --- /dev/null +++ b/src/sample_apps/demo-sample-apps-simple-app-celery.sh @@ -0,0 +1,33 @@ +# If not already inside it, clone the aiotaskq repository and cd into it +[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) + +# Create a new virtual env specifically for the sample apps +python3.10 -m venv ./src/sample_apps/.venv +source ./src/sample_apps/.venv/bin/activate +echo "Using $(python --version)" + +# Start redis and wait for it to be ready +docker-compose up --detach redis +python ./check_redis_ready.py + +# Let's say we want to run the first app (Simple App), which is located +# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like +# to choose your desired sample app. +APP=simple_app + +# Install sample_apps package from local file +python3 -m pip install --upgrade pip +PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +pip install -e ./src/sample_apps + +# Run Celery workers in background and wait for it to be ready +celery -A sample_apps.$APP worker --concurrency 4 & +sleep 2 + +# Run the the sample app +python3.10 -m sample_apps.$APP.app_celery + +# Confirm in the logs if the app is running correctly + +# Kill celery workers that were running in background +ps | grep celery | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM diff --git a/src/sample_apps/demo-sample-apps-simple-app.sh b/src/sample_apps/demo-sample-apps-simple-app.sh new file mode 100755 index 0000000..986c66b --- /dev/null +++ b/src/sample_apps/demo-sample-apps-simple-app.sh @@ -0,0 +1,35 @@ +# If not already inside it, clone the aiotaskq repository and cd into it +[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) + +# Let's say we want to run the first app (Simple App), which is located +# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like +# to choose your desired sample app. +APP=simple_app + +# Create a new virtual env specifically for the sample apps +rm -rf ./src/sample_apps/.venv || echo "" +python3.10 -m venv ./src/sample_apps/.venv +source ./src/sample_apps/.venv/bin/activate +echo "Using $(python --version)" + +# Install sample_apps package from local file +python -m pip install --no-cache-dir --upgrade pip +PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +pip install --no-cache-dir file://$PWD/src/sample_apps + +# Start redis and wait for it to be ready +docker-compose up -d redis +python ./check_redis_ready.py + +# Run aiotaskq workers in background and wait for it be ready +aiotaskq --version +aiotaskq worker sample_apps.$APP --concurrency 4 & +sleep 2 + +# Run the the sample app +python -m sample_apps.$APP.app_aiotaskq + +# Confirm in the logs if the app is running correctly + +# Kill aiotaskq workers that were running in background +ps | grep aiotaskq | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM diff --git a/src/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml index 3107a56..6922034 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -17,5 +17,5 @@ version = "0.0.0" readme = "README.md" description = "A collection of sample apps to test aiotaskq against Celery" -[tool.setuptools] -packages = ["simple_app", "simple_app_implicit_instance"] +[tool.setuptools.packages.find] +include = ["sample_apps*"] diff --git a/src/sample_apps/simple_app/__init__.py b/src/sample_apps/simple_app/__init__.py index 466fc67..e69de29 100644 --- a/src/sample_apps/simple_app/__init__.py +++ b/src/sample_apps/simple_app/__init__.py @@ -1,17 +0,0 @@ -from . import ( - aiotaskq, - app_aiotaskq, - app_celery, - celery, - tasks_aiotaskq, - tasks_celery, -) - -__all__ = [ - "aiotaskq", - "app_aiotaskq", - "app_celery", - "celery", - "tasks_aiotaskq", - "tasks_celery", -] diff --git a/src/sample_apps/simple_app/app_aiotaskq.py b/src/sample_apps/simple_app/app_aiotaskq.py index f3c9ced..acbb06f 100644 --- a/src/sample_apps/simple_app/app_aiotaskq.py +++ b/src/sample_apps/simple_app/app_aiotaskq.py @@ -1,5 +1,6 @@ import asyncio import logging +import os from .tasks_aiotaskq import add, times @@ -20,7 +21,14 @@ async def apply_formula(): async def main(): - logging.basicConfig(level=logging.DEBUG) + log_level = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "FATAL": logging.FATAL, + }[os.environ["LOG_LEVEL"].upper()] + logging.basicConfig(level=log_level) logger.info("Simple App (Aiotaskq)") ret = await apply_formula() logger.info("Result: %s", ret) diff --git a/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/simple_app/app_celery.py index 1dcc5b0..33b75ce 100644 --- a/src/sample_apps/simple_app/app_celery.py +++ b/src/sample_apps/simple_app/app_celery.py @@ -21,7 +21,6 @@ def apply_formula() -> Signature: def main(): - logging.basicConfig(level=logging.DEBUG) logger.info("Simple App (Celery)") ret = apply_formula().apply_async().get() logger.info("Result: %s", ret) diff --git a/src/sample_apps/simple_app/tests.py b/src/sample_apps/simple_app/tests.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/sample_apps/simple_app_implicit_instance/__init__.py b/src/sample_apps/simple_app_implicit_instance/__init__.py index b41c2ac..8b13789 100644 --- a/src/sample_apps/simple_app_implicit_instance/__init__.py +++ b/src/sample_apps/simple_app_implicit_instance/__init__.py @@ -1,4 +1 @@ -from . import app_aiotaskq -from . import celery -__all__ = ["app_aiotaskq", "celery"] diff --git a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py index 357c8c6..d51e5a4 100644 --- a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py +++ b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py @@ -35,7 +35,6 @@ async def apply_formula(): async def main(): - logging.basicConfig(level=logging.DEBUG) logger.info("Simple App (Aiotaskq)") ret = await apply_formula() logger.info("Result: %s", ret) diff --git a/src/sample_apps/simple_app_implicit_instance/app_celery.py b/src/sample_apps/simple_app_implicit_instance/app_celery.py index 5c24a68..e8e1f91 100644 --- a/src/sample_apps/simple_app_implicit_instance/app_celery.py +++ b/src/sample_apps/simple_app_implicit_instance/app_celery.py @@ -37,7 +37,6 @@ def apply_formula() -> Signature: def main(): - logging.basicConfig(level=logging.DEBUG) logger.info("Simple App (Celery)") ret = apply_formula().apply_async().get() logger.info("Result: %s", ret) diff --git a/src/tests/test_docs_sample_apps.py b/src/tests/test_docs_sample_apps.py new file mode 100644 index 0000000..9bd88dc --- /dev/null +++ b/src/tests/test_docs_sample_apps.py @@ -0,0 +1,21 @@ +import subprocess + + +def test_simple_app(): + proc_aiotaskq = subprocess.run( + ["sh", "./demo-sample-apps-simple-app.sh"] + ) + proc_celery = subprocess.run( + ["sh", "./demo-sample-apps-simple-app-celery.sh"] + ) + assert proc_aiotaskq.returncode == proc_celery.returncode == 0 + + +def test_simple_app_implicit_instance(): + proc_aiotaskq = subprocess.run( + ["sh", "./demo-sample-apps-simple-app-implicit-instance.sh"] + ) + proc_celery = subprocess.run( + ["sh", "./demo-sample-apps-simple-app-implicit-instance-celery.sh"] + ) + assert proc_aiotaskq.returncode == proc_celery.returncode == 0 From 884107866501e4f03a596d555ac9deea1a6fd0f2 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 10 Dec 2022 12:43:36 -0500 Subject: [PATCH 24/42] WIP --- .gitignore | 20 ++++----- ...h => demo-sample-apps-simple-app-celery.sh | 25 +++++------ ...e-app.sh => demo-sample-apps-simple-app.sh | 15 +++---- enter_env.sh | 23 ++++++++++ install_dependencies.sh | 27 ++++++++++++ pyproject.toml | 3 +- src/aiotaskq/app.py | 2 +- src/sample_apps/pyproject.template.toml | 2 + .../{ => src/sample_apps}/__init__.py | 0 .../sample_apps}/simple_app/__init__.py | 0 .../sample_apps}/simple_app/aiotaskq.py | 0 .../sample_apps}/simple_app/app_aiotaskq.py | 0 .../sample_apps}/simple_app/app_celery.py | 0 .../sample_apps}/simple_app/celery.py | 0 .../sample_apps}/simple_app/tasks_aiotaskq.py | 0 .../sample_apps}/simple_app/tasks_celery.py | 0 .../simple_app_implicit_instance/__init__.py | 0 .../app_aiotaskq.py | 0 .../app_celery.py | 0 .../simple_app_implicit_instance/celery.py | 0 src/tests/pyproject.template.toml | 42 +++++++++++++++++++ src/tests/test_celery_parity.py | 2 +- test.sh | 15 +++---- 23 files changed, 130 insertions(+), 46 deletions(-) rename src/sample_apps/demo-sample-apps-simple-app-celery.sh => demo-sample-apps-simple-app-celery.sh (63%) rename src/sample_apps/demo-sample-apps-simple-app.sh => demo-sample-apps-simple-app.sh (61%) create mode 100755 enter_env.sh create mode 100755 install_dependencies.sh rename src/sample_apps/{ => src/sample_apps}/__init__.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app/__init__.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app/aiotaskq.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app/app_aiotaskq.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app/app_celery.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app/celery.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app/tasks_aiotaskq.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app/tasks_celery.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app_implicit_instance/__init__.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app_implicit_instance/app_aiotaskq.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app_implicit_instance/app_celery.py (100%) rename src/sample_apps/{ => src/sample_apps}/simple_app_implicit_instance/celery.py (100%) create mode 100644 src/tests/pyproject.template.toml diff --git a/.gitignore b/.gitignore index 9540103..309416b 100644 --- a/.gitignore +++ b/.gitignore @@ -122,16 +122,16 @@ venv.bak/ # Sample Apps -src/sample_apps/.venv/ -src/sample_apps/pyproject.toml -src/sample_apps/.Python -src/sample_apps/build/ -src/sample_apps/develop-eggs/ -src/sample_apps/dist/ -src/sample_apps/eggs/ -src/sample_apps/.eggs/ -src/sample_apps/*.egg-info/ -src/sample_apps/*.egg +src/*/.venv/ +src/*/pyproject.toml +src/*/.Python +src/*/build/ +src/*/develop-eggs/ +src/*/dist/ +src/*/eggs/ +src/*/.eggs/ +src/*/*.egg-info/ +src/*/*.egg # Spyder project settings .spyderproject diff --git a/src/sample_apps/demo-sample-apps-simple-app-celery.sh b/demo-sample-apps-simple-app-celery.sh similarity index 63% rename from src/sample_apps/demo-sample-apps-simple-app-celery.sh rename to demo-sample-apps-simple-app-celery.sh index 1c939d4..bb8b164 100755 --- a/src/sample_apps/demo-sample-apps-simple-app-celery.sh +++ b/demo-sample-apps-simple-app-celery.sh @@ -1,31 +1,28 @@ # If not already inside it, clone the aiotaskq repository and cd into it [[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) -# Create a new virtual env specifically for the sample apps -python3.10 -m venv ./src/sample_apps/.venv -source ./src/sample_apps/.venv/bin/activate -echo "Using $(python --version)" - -# Start redis and wait for it to be ready -docker-compose up --detach redis -python ./check_redis_ready.py - # Let's say we want to run the first app (Simple App), which is located -# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like +# in `./src/sample_apps/src/sample_apps/simple_app/`. Update this env var APP as you'd like # to choose your desired sample app. APP=simple_app +# Enter virtual env specifically for the sample apps +source ./enter_env.sh ./src/sample_apps/.venv + # Install sample_apps package from local file -python3 -m pip install --upgrade pip -PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install -e ./src/sample_apps +./install_dependencies.sh ./src/sample_apps/ + +# Start redis and wait for it to be ready +docker-compose up --detach redis +python ./check_redis_ready.py # Run Celery workers in background and wait for it to be ready +export LOG_LEVEL=${LOG_LEVEL:-INFO} celery -A sample_apps.$APP worker --concurrency 4 & sleep 2 # Run the the sample app -python3.10 -m sample_apps.$APP.app_celery +LOG_LEVEL=INFO python3.10 -m sample_apps.$APP.app_celery # Confirm in the logs if the app is running correctly diff --git a/src/sample_apps/demo-sample-apps-simple-app.sh b/demo-sample-apps-simple-app.sh similarity index 61% rename from src/sample_apps/demo-sample-apps-simple-app.sh rename to demo-sample-apps-simple-app.sh index 986c66b..72e2fb4 100755 --- a/src/sample_apps/demo-sample-apps-simple-app.sh +++ b/demo-sample-apps-simple-app.sh @@ -2,27 +2,22 @@ [[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) # Let's say we want to run the first app (Simple App), which is located -# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like +# in `./src/sample_apps/src/sample_apps/simple_app/`. Update this env var APP as you'd like # to choose your desired sample app. APP=simple_app -# Create a new virtual env specifically for the sample apps -rm -rf ./src/sample_apps/.venv || echo "" -python3.10 -m venv ./src/sample_apps/.venv -source ./src/sample_apps/.venv/bin/activate -echo "Using $(python --version)" +# Enter virtual env specifically for the sample apps +source ./enter_env.sh ./src/sample_apps/.venv # Install sample_apps package from local file -python -m pip install --no-cache-dir --upgrade pip -PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install --no-cache-dir file://$PWD/src/sample_apps +./install_dependencies.sh ./src/sample_apps/ # Start redis and wait for it to be ready docker-compose up -d redis python ./check_redis_ready.py # Run aiotaskq workers in background and wait for it be ready -aiotaskq --version +export LOG_LEVEL=${LOG_LEVEL:-INFO} aiotaskq worker sample_apps.$APP --concurrency 4 & sleep 2 diff --git a/enter_env.sh b/enter_env.sh new file mode 100755 index 0000000..f35a792 --- /dev/null +++ b/enter_env.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +if hash grealpath 2>/dev/null; then + REALPATH="grealpath" +else + REALPATH="realpath" +fi + +env=${1:-.venv} +env=$(realpath $env) +echo env=$env + +if [ -z "$VIRTUAL_ENV" ]; then + echo "Entering virtual environment '$env'" + source $env/bin/activate || exit 1 +else + if [ "$VIRTUAL_ENV" != "$env" ]; then + echo "Switching virtual environment from '$VIRTUAL_ENV' to '$env'" + source $env/bin/activate || exit 1 + else + echo "Already in virtual environment '$VIRTUAL_ENV'" + fi +fi diff --git a/install_dependencies.sh b/install_dependencies.sh new file mode 100755 index 0000000..78d3493 --- /dev/null +++ b/install_dependencies.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +package=${1:-.} + +if [[ "$package" = "." || "$package" = ".[dev]" ]]; then + source ./enter_env.sh .venv + pip install $package +elif [ "$package" = "./src/sample_apps/" ]; then + source ./enter_env.sh ./src/sample_apps/.venv + pip uninstall sample_apps -y + rm -rf src/sample_apps/build/ src/sample_apps/sample_apps.egg-info/ + PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml +elif [ "$package" = "./src/tests/" ]; then + source ./enter_env.sh ./src/tests/.venv + pip uninstall tests aiotaskq sample_apps -y + rm -rf src/tests/build/ src/tests/tests.egg-info/ \ + src/sample_apps/build/ src/sample_apps/sample_apps.egg-info/ \ + src/aiotaskq/build/ src/aiotaskq/aiotaskq.egg-info/ + PROJECT_DIR=$PWD envsubst < ./src/tests/pyproject.template.toml > ./src/tests/pyproject.toml +else + echo Package $package not recognized. Aborting. + exit 1 +fi + +echo "Using $(pip --version)" +echo "Installing $(realpath $package) ..." +pip install --no-cache-dir $package diff --git a/pyproject.toml b/pyproject.toml index 590ad47..b6b67d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,8 @@ aiotaskq = "aiotaskq.__main__:cli" [tool.setuptools.packages.find] where = ["src"] -include = ["aiotaskq*"] +include = ["aiotaskq"] +namespaces = false [tool.black] line-length = 100 diff --git a/src/aiotaskq/app.py b/src/aiotaskq/app.py index 312cfce..4c1c1a0 100644 --- a/src/aiotaskq/app.py +++ b/src/aiotaskq/app.py @@ -5,7 +5,7 @@ import typing as t from .exceptions import AppImportError -from .task import task, Task, P, RT +from .task import Task if t.TYPE_CHECKING: # pragma: no cover from types import ModuleType diff --git a/src/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml index 6922034..1654179 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -18,4 +18,6 @@ readme = "README.md" description = "A collection of sample apps to test aiotaskq against Celery" [tool.setuptools.packages.find] +where = ["src"] include = ["sample_apps*"] +namespaces = false diff --git a/src/sample_apps/__init__.py b/src/sample_apps/src/sample_apps/__init__.py similarity index 100% rename from src/sample_apps/__init__.py rename to src/sample_apps/src/sample_apps/__init__.py diff --git a/src/sample_apps/simple_app/__init__.py b/src/sample_apps/src/sample_apps/simple_app/__init__.py similarity index 100% rename from src/sample_apps/simple_app/__init__.py rename to src/sample_apps/src/sample_apps/simple_app/__init__.py diff --git a/src/sample_apps/simple_app/aiotaskq.py b/src/sample_apps/src/sample_apps/simple_app/aiotaskq.py similarity index 100% rename from src/sample_apps/simple_app/aiotaskq.py rename to src/sample_apps/src/sample_apps/simple_app/aiotaskq.py diff --git a/src/sample_apps/simple_app/app_aiotaskq.py b/src/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py similarity index 100% rename from src/sample_apps/simple_app/app_aiotaskq.py rename to src/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py diff --git a/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/src/sample_apps/simple_app/app_celery.py similarity index 100% rename from src/sample_apps/simple_app/app_celery.py rename to src/sample_apps/src/sample_apps/simple_app/app_celery.py diff --git a/src/sample_apps/simple_app/celery.py b/src/sample_apps/src/sample_apps/simple_app/celery.py similarity index 100% rename from src/sample_apps/simple_app/celery.py rename to src/sample_apps/src/sample_apps/simple_app/celery.py diff --git a/src/sample_apps/simple_app/tasks_aiotaskq.py b/src/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py similarity index 100% rename from src/sample_apps/simple_app/tasks_aiotaskq.py rename to src/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py diff --git a/src/sample_apps/simple_app/tasks_celery.py b/src/sample_apps/src/sample_apps/simple_app/tasks_celery.py similarity index 100% rename from src/sample_apps/simple_app/tasks_celery.py rename to src/sample_apps/src/sample_apps/simple_app/tasks_celery.py diff --git a/src/sample_apps/simple_app_implicit_instance/__init__.py b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py similarity index 100% rename from src/sample_apps/simple_app_implicit_instance/__init__.py rename to src/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py diff --git a/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py similarity index 100% rename from src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py rename to src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py diff --git a/src/sample_apps/simple_app_implicit_instance/app_celery.py b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py similarity index 100% rename from src/sample_apps/simple_app_implicit_instance/app_celery.py rename to src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py diff --git a/src/sample_apps/simple_app_implicit_instance/celery.py b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py similarity index 100% rename from src/sample_apps/simple_app_implicit_instance/celery.py rename to src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py diff --git a/src/tests/pyproject.template.toml b/src/tests/pyproject.template.toml new file mode 100644 index 0000000..1d98e5e --- /dev/null +++ b/src/tests/pyproject.template.toml @@ -0,0 +1,42 @@ +[build-system] +requires = [ + "setuptools >= 42", + "wheel" +] +build-backend = "setuptools.build_meta" + +[project] +requires-python = ">=3.9" +dependencies = [ + "aiotaskq @ file://${PROJECT_DIR}/", + "sample_apps @ file://${PROJECT_DIR}/src/sample_apps/", + "celery >= 5.2.0, < 5.3.0", + "coverage >= 6.4.0, < 6.5.0", + "pytest-asyncio >= 0.19.0, < 0.20.0", + "pylint >= 2.14.0, < 2.15.0", + "pytest >= 7.1.0, < 7.2.0", +] +name = "tests" +version = "0.0.1" +readme = "README.md" +description = "Test" +authors = [ + {name = "Imran Ariffin", email = "ariffin.imran@gmail.com"}, +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +license = { file = "LICENSE" } + +[tool.setuptools.packages.find] +where = ["."] + +[tool.black] +line-length = 100 + +[tool.pytest.ini_options] +markers = [ + "lowlevel: Marks tests as low-level", +] diff --git a/src/tests/test_celery_parity.py b/src/tests/test_celery_parity.py index 156b5ec..2cc0146 100644 --- a/src/tests/test_celery_parity.py +++ b/src/tests/test_celery_parity.py @@ -55,7 +55,7 @@ async def test_parity_with_celery__simple_app_implicit_instance( ): # Given a simple app that uses both Celery and aiotaskq in # the following file structure (Please check it out): - # > tree src/sample_apps/simple_app_implicit -I __pycache__ + # > tree src/sample_apps/src/sample_apps/simple_app_implicit -I __pycache__ # src/sample_apps/simple_app_implicit_instance # ├── app_aiotaskq.py # ├── app_celery.py diff --git a/test.sh b/test.sh index 7d14973..a6272f4 100755 --- a/test.sh +++ b/test.sh @@ -1,12 +1,9 @@ -echo "Upgrade pip" -python -m pip install --quiet --upgrade pip - -echo "Install dependencies" -pip install --quiet -e .[dev] - -echo "Install sample apps" -PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install --quiet -e file://$PWD/src/sample_apps/ +echo "Enter virtual environment for testing" +source ./enter_env.sh ./src/tests/.venv +source ./install_dependencies.sh ./src/tests/ +# source ./enter_env.sh .[dev] +# source ./enter_env.sh ./src/sample_apps/.venv +echo "Using $(pip --version)" echo "Erase previous coverage files" coverage erase From ff991e9e3aca9c6364cd7026834dcb7c2c08616c Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 17 Dec 2022 22:23:20 -0500 Subject: [PATCH 25/42] W I P --- ...=> demo-sample-apps-simple-app-aiotaskq.sh | 0 src/sample_apps/README.md | 2 +- .../src/sample_apps/simple_app/app_celery.py | 9 +++++++ .../app_celery.py | 20 +++------------ .../simple_app_implicit_instance/celery.py | 25 ++++++++++++++++++- src/tests/test_docs_sample_apps.py | 12 +-------- test.sh | 1 + 7 files changed, 39 insertions(+), 30 deletions(-) rename demo-sample-apps-simple-app.sh => demo-sample-apps-simple-app-aiotaskq.sh (100%) diff --git a/demo-sample-apps-simple-app.sh b/demo-sample-apps-simple-app-aiotaskq.sh similarity index 100% rename from demo-sample-apps-simple-app.sh rename to demo-sample-apps-simple-app-aiotaskq.sh diff --git a/src/sample_apps/README.md b/src/sample_apps/README.md index 2774bcf..479e1a5 100644 --- a/src/sample_apps/README.md +++ b/src/sample_apps/README.md @@ -28,7 +28,7 @@ Feel free to run this inside of a `aiotaskq` repository. Copy-pasting these commands to your terminal should work out of the box. ```bash -# ./demo-sample-apps-simple-app.sh#L1-L35 +# ./demo-sample-apps-simple-app-aiotaskq.sh#L1-L35 # If not already inside it, clone the aiotaskq repository and cd into it [[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) diff --git a/src/sample_apps/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/src/sample_apps/simple_app/app_celery.py index 33b75ce..c0000ca 100644 --- a/src/sample_apps/src/sample_apps/simple_app/app_celery.py +++ b/src/sample_apps/src/sample_apps/simple_app/app_celery.py @@ -1,4 +1,5 @@ import logging +import os from celery.canvas import Signature, chord, group from .tasks_celery import add, times @@ -21,6 +22,14 @@ def apply_formula() -> Signature: def main(): + log_level = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "FATAL": logging.FATAL, + }[os.environ["LOG_LEVEL"].upper()] + logging.basicConfig(level=log_level) logger.info("Simple App (Celery)") ret = apply_formula().apply_async().get() logger.info("Result: %s", ret) diff --git a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py index e8e1f91..a03293b 100644 --- a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py +++ b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py @@ -1,25 +1,11 @@ import logging -from celery import shared_task -from celery.canvas import Signature, chord, group - -logger = logging.getLogger(__name__) - -@shared_task -def add(ls: list[int]) -> int: - logger.info("add(%s) ...", ls) - ret = sum(x for x in ls) - logger.info("add(%s) -> %s", ls, ret) - return ret +from celery.canvas import Signature, chord, group +from .celery import times, add -@shared_task -def times(x: int, y: int) -> int: - logger.info("times(%s, %s) ...", x, y) - ret = x * y - logger.info("times(%s, %s) -> %s", x, y, ret) - return ret +logger = logging.getLogger(__name__) def apply_formula() -> Signature: diff --git a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py index b0b35bd..dfecf65 100644 --- a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py +++ b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py @@ -1,5 +1,28 @@ +import logging + from celery import Celery +from celery import shared_task + +logger = logging.getLogger(__name__) -app = Celery() +app = Celery( + include=["sample_apps.simple_app_implicit_instance.app_celery"], +) app.conf.broker_url = "redis://localhost:6379/0" app.conf.result_backend = "redis://localhost:6379/0" + + +@shared_task +def add(ls: list[int]) -> int: + logger.debug("add(%s) ...", ls) + ret = sum(x for x in ls) + logger.debug("add(%s) -> %s", ls, ret) + return ret + + +@shared_task +def times(x: int, y: int) -> int: + logger.debug("times(%s, %s) ...", x, y) + ret = x * y + logger.debug("times(%s, %s) -> %s", x, y, ret) + return ret diff --git a/src/tests/test_docs_sample_apps.py b/src/tests/test_docs_sample_apps.py index 9bd88dc..df734b9 100644 --- a/src/tests/test_docs_sample_apps.py +++ b/src/tests/test_docs_sample_apps.py @@ -3,19 +3,9 @@ def test_simple_app(): proc_aiotaskq = subprocess.run( - ["sh", "./demo-sample-apps-simple-app.sh"] + ["sh", "./demo-sample-apps-simple-app-aiotaskq.sh"] ) proc_celery = subprocess.run( ["sh", "./demo-sample-apps-simple-app-celery.sh"] ) assert proc_aiotaskq.returncode == proc_celery.returncode == 0 - - -def test_simple_app_implicit_instance(): - proc_aiotaskq = subprocess.run( - ["sh", "./demo-sample-apps-simple-app-implicit-instance.sh"] - ) - proc_celery = subprocess.run( - ["sh", "./demo-sample-apps-simple-app-implicit-instance-celery.sh"] - ) - assert proc_aiotaskq.returncode == proc_celery.returncode == 0 diff --git a/test.sh b/test.sh index a6272f4..4d641e5 100755 --- a/test.sh +++ b/test.sh @@ -3,6 +3,7 @@ source ./enter_env.sh ./src/tests/.venv source ./install_dependencies.sh ./src/tests/ # source ./enter_env.sh .[dev] # source ./enter_env.sh ./src/sample_apps/.venv +export LOG_LEVEL=${LOG_LEVEL:-INFO} echo "Using $(pip --version)" echo "Erase previous coverage files" From ee664e47c7fafefbdd5504e37980571c3c9fb756 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 17 Dec 2022 22:40:13 -0500 Subject: [PATCH 26/42] Fixup - Pass all tests --- demo-sample-apps-simple-app-aiotaskq.sh | 5 ++++- demo-sample-apps-simple-app-celery.sh | 5 ++++- test.sh | 2 -- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/demo-sample-apps-simple-app-aiotaskq.sh b/demo-sample-apps-simple-app-aiotaskq.sh index 72e2fb4..25a3c06 100755 --- a/demo-sample-apps-simple-app-aiotaskq.sh +++ b/demo-sample-apps-simple-app-aiotaskq.sh @@ -1,5 +1,8 @@ # If not already inside it, clone the aiotaskq repository and cd into it -[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) +if [ "$(basename $PWD)" != "aiotaskq" ]; then + git clone git@github.com:imranariffin/aiotaskq.git; + cd aiotaskq +fi # Let's say we want to run the first app (Simple App), which is located # in `./src/sample_apps/src/sample_apps/simple_app/`. Update this env var APP as you'd like diff --git a/demo-sample-apps-simple-app-celery.sh b/demo-sample-apps-simple-app-celery.sh index bb8b164..394e8a7 100755 --- a/demo-sample-apps-simple-app-celery.sh +++ b/demo-sample-apps-simple-app-celery.sh @@ -1,5 +1,8 @@ # If not already inside it, clone the aiotaskq repository and cd into it -[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) +if [ "$(basename $PWD)" != "aiotaskq" ]; then + git clone git@github.com:imranariffin/aiotaskq.git; + cd aiotaskq +fi # Let's say we want to run the first app (Simple App), which is located # in `./src/sample_apps/src/sample_apps/simple_app/`. Update this env var APP as you'd like diff --git a/test.sh b/test.sh index 4d641e5..7ffee7b 100755 --- a/test.sh +++ b/test.sh @@ -1,8 +1,6 @@ echo "Enter virtual environment for testing" source ./enter_env.sh ./src/tests/.venv source ./install_dependencies.sh ./src/tests/ -# source ./enter_env.sh .[dev] -# source ./enter_env.sh ./src/sample_apps/.venv export LOG_LEVEL=${LOG_LEVEL:-INFO} echo "Using $(pip --version)" From 2e34a0c0b6493b8c016455c81fa40119aa597356 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 17 Dec 2022 22:46:43 -0500 Subject: [PATCH 27/42] Fixup: demo scripts updated on sample_apps README --- src/sample_apps/README.md | 54 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/src/sample_apps/README.md b/src/sample_apps/README.md index 479e1a5..54f1270 100644 --- a/src/sample_apps/README.md +++ b/src/sample_apps/README.md @@ -28,33 +28,31 @@ Feel free to run this inside of a `aiotaskq` repository. Copy-pasting these commands to your terminal should work out of the box. ```bash -# ./demo-sample-apps-simple-app-aiotaskq.sh#L1-L35 +# ../../demo-sample-apps-simple-app-aiotaskq.sh#L1-L33 # If not already inside it, clone the aiotaskq repository and cd into it -[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) +if [ "$(basename $PWD)" != "aiotaskq" ]; then + git clone git@github.com:imranariffin/aiotaskq.git; + cd aiotaskq +fi # Let's say we want to run the first app (Simple App), which is located -# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like +# in `./src/sample_apps/src/sample_apps/simple_app/`. Update this env var APP as you'd like # to choose your desired sample app. APP=simple_app -# Create a new virtual env specifically for the sample apps -rm -rf ./src/sample_apps/.venv || echo "" -python3.10 -m venv ./src/sample_apps/.venv -source ./src/sample_apps/.venv/bin/activate -echo "Using $(python --version)" +# Enter virtual env specifically for the sample apps +source ./enter_env.sh ./src/sample_apps/.venv # Install sample_apps package from local file -python -m pip install --no-cache-dir --upgrade pip -PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install --no-cache-dir file://$PWD/src/sample_apps +./install_dependencies.sh ./src/sample_apps/ # Start redis and wait for it to be ready docker-compose up -d redis python ./check_redis_ready.py # Run aiotaskq workers in background and wait for it be ready -aiotaskq --version +export LOG_LEVEL=${LOG_LEVEL:-INFO} aiotaskq worker sample_apps.$APP --concurrency 4 & sleep 2 @@ -73,36 +71,36 @@ Now, if you want to run the sample app with `Celery` to see how `aiotaskq` compares to it, do the following: ```bash -# ./demo-sample-apps-simple-app-celery.sh#L1-L33 +# ../../demo-sample-apps-simple-app-celery.sh#L1-L33 # If not already inside it, clone the aiotaskq repository and cd into it -[[ $(basename $PWD) == "aiotaskq" ]] || (git clone git@github.com:imranariffin/aiotaskq.git && cd aiotaskq) - -# Create a new virtual env specifically for the sample apps -python3.10 -m venv ./src/sample_apps/.venv -source ./src/sample_apps/.venv/bin/activate -echo "Using $(python --version)" - -# Start redis and wait for it to be ready -docker-compose up --detach redis -python ./check_redis_ready.py +if [ "$(basename $PWD)" != "aiotaskq" ]; then + git clone git@github.com:imranariffin/aiotaskq.git; + cd aiotaskq +fi # Let's say we want to run the first app (Simple App), which is located -# in `./src/sample_apps/simple_app/`. Update this env var APP as you'd like +# in `./src/sample_apps/src/sample_apps/simple_app/`. Update this env var APP as you'd like # to choose your desired sample app. APP=simple_app +# Enter virtual env specifically for the sample apps +source ./enter_env.sh ./src/sample_apps/.venv + # Install sample_apps package from local file -python3 -m pip install --upgrade pip -PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml -pip install -e ./src/sample_apps +./install_dependencies.sh ./src/sample_apps/ + +# Start redis and wait for it to be ready +docker-compose up --detach redis +python ./check_redis_ready.py # Run Celery workers in background and wait for it to be ready +export LOG_LEVEL=${LOG_LEVEL:-INFO} celery -A sample_apps.$APP worker --concurrency 4 & sleep 2 # Run the the sample app -python3.10 -m sample_apps.$APP.app_celery +LOG_LEVEL=INFO python3.10 -m sample_apps.$APP.app_celery # Confirm in the logs if the app is running correctly From a8d7463baab85ea8e2478e55c05026ca8aacbd36 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sat, 17 Dec 2022 22:56:19 -0500 Subject: [PATCH 28/42] Fixup --- src/sample_apps/pyproject.template.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml index 1654179..5e2611b 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -18,6 +18,4 @@ readme = "README.md" description = "A collection of sample apps to test aiotaskq against Celery" [tool.setuptools.packages.find] -where = ["src"] -include = ["sample_apps*"] -namespaces = false +where = ["."] From b5ca71626ad246503bbfe50b93a8b92151f3255c Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 11:53:50 -0500 Subject: [PATCH 29/42] Fixup --- lint.sh | 9 +++++++++ src/sample_apps/{src/sample_apps => }/__init__.py | 0 .../{src/sample_apps => }/simple_app/__init__.py | 0 .../{src/sample_apps => }/simple_app/aiotaskq.py | 0 .../{src/sample_apps => }/simple_app/app_aiotaskq.py | 0 .../{src/sample_apps => }/simple_app/app_celery.py | 0 .../{src/sample_apps => }/simple_app/celery.py | 0 .../{src/sample_apps => }/simple_app/tasks_aiotaskq.py | 0 .../{src/sample_apps => }/simple_app/tasks_celery.py | 0 src/sample_apps/simple_app_implicit_instance/__init__.py | 0 .../simple_app_implicit_instance/app_aiotaskq.py | 0 .../simple_app_implicit_instance/app_celery.py | 0 .../simple_app_implicit_instance/celery.py | 0 .../sample_apps/simple_app_implicit_instance/__init__.py | 1 - src/tests/test_docs_sample_apps.py | 6 ++++-- 15 files changed, 13 insertions(+), 3 deletions(-) rename src/sample_apps/{src/sample_apps => }/__init__.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app/__init__.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app/aiotaskq.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app/app_aiotaskq.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app/app_celery.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app/celery.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app/tasks_aiotaskq.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app/tasks_celery.py (100%) create mode 100644 src/sample_apps/simple_app_implicit_instance/__init__.py rename src/sample_apps/{src/sample_apps => }/simple_app_implicit_instance/app_aiotaskq.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app_implicit_instance/app_celery.py (100%) rename src/sample_apps/{src/sample_apps => }/simple_app_implicit_instance/celery.py (100%) delete mode 100644 src/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py diff --git a/lint.sh b/lint.sh index aac843c..9d09595 100755 --- a/lint.sh +++ b/lint.sh @@ -1,11 +1,20 @@ if [ -z $1 ]; then + source ./enter_env.sh + echo "Installing dependencies" + source ./install_dependencies.sh > /dev/null pylint -v --rcfile ./.pylintrc src/aiotaskq failed_1=$? + source ./enter_env.sh ./src/tests/.venv/ + echo "Installing dependencies" + source ./install_dependencies.sh ./src/tests/ > /dev/null pylint -v --rcfile ./.pylintrc.tests src/tests/ failed_2=$? + source ./enter_env.sh ./src/sample_apps/.venv/ + echo "Installing dependencies" + source ./install_dependencies.sh ./src/sample_apps/ > /dev/null pylint -v --rcfile ./.pylintrc.sample_apps src/sample_apps/ failed_3=$? diff --git a/src/sample_apps/src/sample_apps/__init__.py b/src/sample_apps/__init__.py similarity index 100% rename from src/sample_apps/src/sample_apps/__init__.py rename to src/sample_apps/__init__.py diff --git a/src/sample_apps/src/sample_apps/simple_app/__init__.py b/src/sample_apps/simple_app/__init__.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app/__init__.py rename to src/sample_apps/simple_app/__init__.py diff --git a/src/sample_apps/src/sample_apps/simple_app/aiotaskq.py b/src/sample_apps/simple_app/aiotaskq.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app/aiotaskq.py rename to src/sample_apps/simple_app/aiotaskq.py diff --git a/src/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py b/src/sample_apps/simple_app/app_aiotaskq.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app/app_aiotaskq.py rename to src/sample_apps/simple_app/app_aiotaskq.py diff --git a/src/sample_apps/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/simple_app/app_celery.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app/app_celery.py rename to src/sample_apps/simple_app/app_celery.py diff --git a/src/sample_apps/src/sample_apps/simple_app/celery.py b/src/sample_apps/simple_app/celery.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app/celery.py rename to src/sample_apps/simple_app/celery.py diff --git a/src/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py b/src/sample_apps/simple_app/tasks_aiotaskq.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app/tasks_aiotaskq.py rename to src/sample_apps/simple_app/tasks_aiotaskq.py diff --git a/src/sample_apps/src/sample_apps/simple_app/tasks_celery.py b/src/sample_apps/simple_app/tasks_celery.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app/tasks_celery.py rename to src/sample_apps/simple_app/tasks_celery.py diff --git a/src/sample_apps/simple_app_implicit_instance/__init__.py b/src/sample_apps/simple_app_implicit_instance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py b/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py rename to src/sample_apps/simple_app_implicit_instance/app_aiotaskq.py diff --git a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py b/src/sample_apps/simple_app_implicit_instance/app_celery.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app_implicit_instance/app_celery.py rename to src/sample_apps/simple_app_implicit_instance/app_celery.py diff --git a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py b/src/sample_apps/simple_app_implicit_instance/celery.py similarity index 100% rename from src/sample_apps/src/sample_apps/simple_app_implicit_instance/celery.py rename to src/sample_apps/simple_app_implicit_instance/celery.py diff --git a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py b/src/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/src/sample_apps/src/sample_apps/simple_app_implicit_instance/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/tests/test_docs_sample_apps.py b/src/tests/test_docs_sample_apps.py index df734b9..f135d4d 100644 --- a/src/tests/test_docs_sample_apps.py +++ b/src/tests/test_docs_sample_apps.py @@ -3,9 +3,11 @@ def test_simple_app(): proc_aiotaskq = subprocess.run( - ["sh", "./demo-sample-apps-simple-app-aiotaskq.sh"] + ["sh", "./demo-sample-apps-simple-app-aiotaskq.sh"], + check=False, ) proc_celery = subprocess.run( - ["sh", "./demo-sample-apps-simple-app-celery.sh"] + ["sh", "./demo-sample-apps-simple-app-celery.sh"], + check=False, ) assert proc_aiotaskq.returncode == proc_celery.returncode == 0 From 7c052460444a452cb173c30ae490e51c0b0d3c04 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 12:38:39 -0500 Subject: [PATCH 30/42] Fixup --- .github/workflows/build.yaml | 7 ++++--- .github/workflows/coverage.yaml | 7 ++++--- .github/workflows/pylint.yaml | 4 ---- enter_env.sh => env_activate.sh | 0 activate.sh => env_create.sh | 12 +++++------- install_dependencies.sh | 13 +++++-------- lint.sh | 9 ++++++--- test.sh | 4 +--- 8 files changed, 25 insertions(+), 31 deletions(-) rename enter_env.sh => env_activate.sh (100%) rename activate.sh => env_create.sh (51%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 231939d..ec74756 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,8 +16,9 @@ jobs: - uses: actions/setup-python@v3 with: python-version: "3.10.x" - - run: source ./activate.sh - - run: pip install --upgrade pip - - run: pip install -e .[dev] + - run: ./env_create.sh ./src/tests/.venv/ + - run: source ./env_activate.sh ./src/tests/.venv/ + - run: ./install_dependencies.sh ./src/tests/ + - run: python ./check_redis_ready.py - run: pip freeze - run: ./test.sh diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 760e9dc..24c4249 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -16,9 +16,10 @@ jobs: - uses: actions/setup-python@v3 with: python-version: "3.10.x" - - run: source ./activate.sh - - run: pip install --upgrade pip - - run: pip install -e .[dev] + - run: ./env_create.sh ./src/tests/.venv/ + - run: source ./env_activate.sh ./src/tests/.venv/ + - run: ./install_dependencies.sh ./src/tests/ + - run: python ./check_redis_ready.py - run: pip freeze - run: ./test.sh - uses: codecov/codecov-action@v3.1.0 diff --git a/.github/workflows/pylint.yaml b/.github/workflows/pylint.yaml index 9c6c988..94dc5c1 100644 --- a/.github/workflows/pylint.yaml +++ b/.github/workflows/pylint.yaml @@ -15,8 +15,4 @@ jobs: - uses: actions/setup-python@v3 with: python-version: "3.10.x" - - run: source ./activate.sh - - run: pip install --upgrade pip - - run: pip install -e .[dev] - - run: pip freeze - run: ./lint.sh diff --git a/enter_env.sh b/env_activate.sh similarity index 100% rename from enter_env.sh rename to env_activate.sh diff --git a/activate.sh b/env_create.sh similarity index 51% rename from activate.sh rename to env_create.sh index 3129db9..a503fcb 100755 --- a/activate.sh +++ b/env_create.sh @@ -7,11 +7,9 @@ if [ "$py_version" != "3.10" ]; then exit 1 fi -python3 -m venv .venv -source .venv/bin/activate +venv=${1:-.venv} +python3 -m venv $venv +source $venv/bin/activate -pip install --upgrade pip --quiet -pip install -e .[dev] --quiet - -python --version -pip --version +echo "Using $(python --version)" +echo "Using $(pip --version)" diff --git a/install_dependencies.sh b/install_dependencies.sh index 78d3493..f642f0a 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -3,25 +3,22 @@ package=${1:-.} if [[ "$package" = "." || "$package" = ".[dev]" ]]; then - source ./enter_env.sh .venv - pip install $package + : elif [ "$package" = "./src/sample_apps/" ]; then - source ./enter_env.sh ./src/sample_apps/.venv pip uninstall sample_apps -y - rm -rf src/sample_apps/build/ src/sample_apps/sample_apps.egg-info/ - PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml + rm -rf ${package}build/ ${package}/sample_apps.egg-info/ + PROJECT_DIR=$PWD envsubst < ${package}/pyproject.template.toml > ${package}/pyproject.toml elif [ "$package" = "./src/tests/" ]; then - source ./enter_env.sh ./src/tests/.venv pip uninstall tests aiotaskq sample_apps -y rm -rf src/tests/build/ src/tests/tests.egg-info/ \ src/sample_apps/build/ src/sample_apps/sample_apps.egg-info/ \ src/aiotaskq/build/ src/aiotaskq/aiotaskq.egg-info/ PROJECT_DIR=$PWD envsubst < ./src/tests/pyproject.template.toml > ./src/tests/pyproject.toml else - echo Package $package not recognized. Aborting. - exit 1 + echo Custom package $package detected. fi echo "Using $(pip --version)" echo "Installing $(realpath $package) ..." +pip install --upgrade pip pip install --no-cache-dir $package diff --git a/lint.sh b/lint.sh index 9d09595..a892ff9 100755 --- a/lint.sh +++ b/lint.sh @@ -1,18 +1,21 @@ if [ -z $1 ]; then - source ./enter_env.sh + ./env_create.sh + source ./env_activate.sh echo "Installing dependencies" source ./install_dependencies.sh > /dev/null pylint -v --rcfile ./.pylintrc src/aiotaskq failed_1=$? - source ./enter_env.sh ./src/tests/.venv/ + ./env_create.sh ./src/tests/.venv/ + source ./env_activate.sh ./src/tests/.venv/ echo "Installing dependencies" source ./install_dependencies.sh ./src/tests/ > /dev/null pylint -v --rcfile ./.pylintrc.tests src/tests/ failed_2=$? - source ./enter_env.sh ./src/sample_apps/.venv/ + ./env_create.sh ./src/sample_apps/.venv/ + source ./env_activate.sh ./src/sample_apps/.venv/ echo "Installing dependencies" source ./install_dependencies.sh ./src/sample_apps/ > /dev/null pylint -v --rcfile ./.pylintrc.sample_apps src/sample_apps/ diff --git a/test.sh b/test.sh index 7ffee7b..c85fd4f 100755 --- a/test.sh +++ b/test.sh @@ -1,8 +1,6 @@ echo "Enter virtual environment for testing" -source ./enter_env.sh ./src/tests/.venv -source ./install_dependencies.sh ./src/tests/ +source ./env_activate.sh ./src/tests/.venv export LOG_LEVEL=${LOG_LEVEL:-INFO} -echo "Using $(pip --version)" echo "Erase previous coverage files" coverage erase From e1e2cd4a312569b9fd154681bb9434cd3508578f Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 12:43:18 -0500 Subject: [PATCH 31/42] Fixup --- install_dependencies.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_dependencies.sh b/install_dependencies.sh index f642f0a..b38019f 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -13,6 +13,7 @@ elif [ "$package" = "./src/tests/" ]; then rm -rf src/tests/build/ src/tests/tests.egg-info/ \ src/sample_apps/build/ src/sample_apps/sample_apps.egg-info/ \ src/aiotaskq/build/ src/aiotaskq/aiotaskq.egg-info/ + PROJECT_DIR=$PWD envsubst < ./src/sample_apps/pyproject.template.toml > ./src/sample_apps/pyproject.toml PROJECT_DIR=$PWD envsubst < ./src/tests/pyproject.template.toml > ./src/tests/pyproject.toml else echo Custom package $package detected. From 11fd695bbbc301b9a4781092b1df54240e8cb3d3 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 12:49:00 -0500 Subject: [PATCH 32/42] Fixup --- src/sample_apps/pyproject.template.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sample_apps/pyproject.template.toml b/src/sample_apps/pyproject.template.toml index 5e2611b..8c62478 100644 --- a/src/sample_apps/pyproject.template.toml +++ b/src/sample_apps/pyproject.template.toml @@ -10,6 +10,7 @@ requires-python = ">=3.9" dependencies = [ "aiotaskq @ file://${PROJECT_DIR}/", "celery >= 5.2.0, < 5.3.0", + "pylint >= 2.14.5, < 2.15.0", "redis >= 4.3.4, < 4.4.0", ] name = "sample_apps" From ece97ac872232ecdedd8f5f92aa20736b8dbfa5d Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 13:31:52 -0500 Subject: [PATCH 33/42] Fixup tests --- demo-sample-apps-simple-app-aiotaskq.sh | 2 +- demo-sample-apps-simple-app-celery.sh | 2 +- lint.sh | 8 ++++---- src/sample_apps/README.md | 4 ++-- src/tests/test_docs_sample_apps.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/demo-sample-apps-simple-app-aiotaskq.sh b/demo-sample-apps-simple-app-aiotaskq.sh index 25a3c06..e76bde1 100755 --- a/demo-sample-apps-simple-app-aiotaskq.sh +++ b/demo-sample-apps-simple-app-aiotaskq.sh @@ -10,7 +10,7 @@ fi APP=simple_app # Enter virtual env specifically for the sample apps -source ./enter_env.sh ./src/sample_apps/.venv +source ./env_activate.sh ./src/sample_apps/.venv # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ diff --git a/demo-sample-apps-simple-app-celery.sh b/demo-sample-apps-simple-app-celery.sh index 394e8a7..7c06366 100755 --- a/demo-sample-apps-simple-app-celery.sh +++ b/demo-sample-apps-simple-app-celery.sh @@ -10,7 +10,7 @@ fi APP=simple_app # Enter virtual env specifically for the sample apps -source ./enter_env.sh ./src/sample_apps/.venv +source ./env_activate.sh ./src/sample_apps/.venv # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ diff --git a/lint.sh b/lint.sh index a892ff9..58ecf23 100755 --- a/lint.sh +++ b/lint.sh @@ -1,20 +1,20 @@ if [ -z $1 ]; then - ./env_create.sh + source ./env_create.sh || exit 1 source ./env_activate.sh echo "Installing dependencies" - source ./install_dependencies.sh > /dev/null + source ./install_dependencies.sh .[dev] > /dev/null pylint -v --rcfile ./.pylintrc src/aiotaskq failed_1=$? - ./env_create.sh ./src/tests/.venv/ + source ./env_create.sh ./src/tests/.venv/ || exit 1 source ./env_activate.sh ./src/tests/.venv/ echo "Installing dependencies" source ./install_dependencies.sh ./src/tests/ > /dev/null pylint -v --rcfile ./.pylintrc.tests src/tests/ failed_2=$? - ./env_create.sh ./src/sample_apps/.venv/ + source ./env_create.sh ./src/sample_apps/.venv/ || exit 1 source ./env_activate.sh ./src/sample_apps/.venv/ echo "Installing dependencies" source ./install_dependencies.sh ./src/sample_apps/ > /dev/null diff --git a/src/sample_apps/README.md b/src/sample_apps/README.md index 54f1270..8e07519 100644 --- a/src/sample_apps/README.md +++ b/src/sample_apps/README.md @@ -42,7 +42,7 @@ fi APP=simple_app # Enter virtual env specifically for the sample apps -source ./enter_env.sh ./src/sample_apps/.venv +source ./env_activate.sh ./src/sample_apps/.venv # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ @@ -85,7 +85,7 @@ fi APP=simple_app # Enter virtual env specifically for the sample apps -source ./enter_env.sh ./src/sample_apps/.venv +source ./env_activate.sh ./src/sample_apps/.venv # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ diff --git a/src/tests/test_docs_sample_apps.py b/src/tests/test_docs_sample_apps.py index f135d4d..ee6702b 100644 --- a/src/tests/test_docs_sample_apps.py +++ b/src/tests/test_docs_sample_apps.py @@ -3,11 +3,11 @@ def test_simple_app(): proc_aiotaskq = subprocess.run( - ["sh", "./demo-sample-apps-simple-app-aiotaskq.sh"], + ["bash", "./demo-sample-apps-simple-app-aiotaskq.sh"], check=False, ) proc_celery = subprocess.run( - ["sh", "./demo-sample-apps-simple-app-celery.sh"], + ["bash", "./demo-sample-apps-simple-app-celery.sh"], check=False, ) assert proc_aiotaskq.returncode == proc_celery.returncode == 0 From 88cbca6f92962ddb7063b5cccc19ae22078cb27b Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 13:49:18 -0500 Subject: [PATCH 34/42] Fixup tests --- .github/workflows/build.yaml | 1 + .vscode/settings.json | 2 +- test.sh | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ec74756..3e4664d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,6 +17,7 @@ jobs: with: python-version: "3.10.x" - run: ./env_create.sh ./src/tests/.venv/ + - run: ./env_create.sh ./src/sample_apps/.venv/ - run: source ./env_activate.sh ./src/tests/.venv/ - run: ./install_dependencies.sh ./src/tests/ - run: python ./check_redis_ready.py diff --git a/.vscode/settings.json b/.vscode/settings.json index a6cce3c..57c6334 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,7 @@ "python.testing.unittestEnabled": false, "python.analysis.extraPaths": [ "src/tests/", - "src/sample_apps" + "src/sample_apps/" ], "python.defaultInterpreterPath": "../.venv/bin/python3" } diff --git a/test.sh b/test.sh index c85fd4f..90641f6 100755 --- a/test.sh +++ b/test.sh @@ -1,6 +1,7 @@ echo "Enter virtual environment for testing" source ./env_activate.sh ./src/tests/.venv export LOG_LEVEL=${LOG_LEVEL:-INFO} +python --version echo "Erase previous coverage files" coverage erase @@ -8,7 +9,7 @@ coverage erase echo "Run tests" if [ -z $1 ]; then - coverage run -m pytest -v + coverage run -m pytest -vvv -s else coverage run -m pytest -vvv -k $1 -s fi From a5e9023f1f110c4a0895cf32512bc456d85c2209 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 14:20:57 -0500 Subject: [PATCH 35/42] Debug --- .github/workflows/build.yaml | 2 +- demo-sample-apps-simple-app-aiotaskq.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3e4664d..3b90448 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,9 +17,9 @@ jobs: with: python-version: "3.10.x" - run: ./env_create.sh ./src/tests/.venv/ - - run: ./env_create.sh ./src/sample_apps/.venv/ - run: source ./env_activate.sh ./src/tests/.venv/ - run: ./install_dependencies.sh ./src/tests/ - run: python ./check_redis_ready.py - run: pip freeze + - run: ./env_create.sh ./src/sample_apps/.venv/ - run: ./test.sh diff --git a/demo-sample-apps-simple-app-aiotaskq.sh b/demo-sample-apps-simple-app-aiotaskq.sh index e76bde1..b0c7696 100755 --- a/demo-sample-apps-simple-app-aiotaskq.sh +++ b/demo-sample-apps-simple-app-aiotaskq.sh @@ -20,6 +20,9 @@ docker-compose up -d redis python ./check_redis_ready.py # Run aiotaskq workers in background and wait for it be ready +echo "DEBUG START" +echo "Using $(python --version) from $(which python)" +echo "DEBUG END" export LOG_LEVEL=${LOG_LEVEL:-INFO} aiotaskq worker sample_apps.$APP --concurrency 4 & sleep 2 From f82a7687f23c47fd255af90c0355168285b6f0b5 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 14:26:35 -0500 Subject: [PATCH 36/42] Debug 2 --- demo-sample-apps-simple-app-aiotaskq.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/demo-sample-apps-simple-app-aiotaskq.sh b/demo-sample-apps-simple-app-aiotaskq.sh index b0c7696..4173d9a 100755 --- a/demo-sample-apps-simple-app-aiotaskq.sh +++ b/demo-sample-apps-simple-app-aiotaskq.sh @@ -22,6 +22,7 @@ python ./check_redis_ready.py # Run aiotaskq workers in background and wait for it be ready echo "DEBUG START" echo "Using $(python --version) from $(which python)" +pip freeze echo "DEBUG END" export LOG_LEVEL=${LOG_LEVEL:-INFO} aiotaskq worker sample_apps.$APP --concurrency 4 & From 473eba2dc17cf5888a835cb5415dcbdb631b6189 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 16:00:29 -0500 Subject: [PATCH 37/42] Fixup --- demo-sample-apps-simple-app-aiotaskq.sh | 5 +++-- demo-sample-apps-simple-app-celery.sh | 5 +++-- src/sample_apps/README.md | 20 ++++++++++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/demo-sample-apps-simple-app-aiotaskq.sh b/demo-sample-apps-simple-app-aiotaskq.sh index 4173d9a..9bd42f3 100755 --- a/demo-sample-apps-simple-app-aiotaskq.sh +++ b/demo-sample-apps-simple-app-aiotaskq.sh @@ -9,8 +9,9 @@ fi # to choose your desired sample app. APP=simple_app -# Enter virtual env specifically for the sample apps -source ./env_activate.sh ./src/sample_apps/.venv +# Create and activate virtual env specifically for the sample apps +source ./env_create.sh ./src/sample_apps/.venv/ +source ./env_activate.sh ./src/sample_apps/.venv/ # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ diff --git a/demo-sample-apps-simple-app-celery.sh b/demo-sample-apps-simple-app-celery.sh index 7c06366..cdacd58 100755 --- a/demo-sample-apps-simple-app-celery.sh +++ b/demo-sample-apps-simple-app-celery.sh @@ -9,8 +9,9 @@ fi # to choose your desired sample app. APP=simple_app -# Enter virtual env specifically for the sample apps -source ./env_activate.sh ./src/sample_apps/.venv +# Create and activate virtual env specifically for the sample apps +source ./env_create.sh ./src/sample_apps/.venv/ +source ./env_activate.sh ./src/sample_apps/.venv/ # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ diff --git a/src/sample_apps/README.md b/src/sample_apps/README.md index 8e07519..a47a9d2 100644 --- a/src/sample_apps/README.md +++ b/src/sample_apps/README.md @@ -28,7 +28,7 @@ Feel free to run this inside of a `aiotaskq` repository. Copy-pasting these commands to your terminal should work out of the box. ```bash -# ../../demo-sample-apps-simple-app-aiotaskq.sh#L1-L33 +# ../../demo-sample-apps-simple-app-aiotaskq.sh # If not already inside it, clone the aiotaskq repository and cd into it if [ "$(basename $PWD)" != "aiotaskq" ]; then @@ -41,8 +41,9 @@ fi # to choose your desired sample app. APP=simple_app -# Enter virtual env specifically for the sample apps -source ./env_activate.sh ./src/sample_apps/.venv +# Create and activate virtual env specifically for the sample apps +source ./env_create.sh ./src/sample_apps/.venv/ +source ./env_activate.sh ./src/sample_apps/.venv/ # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ @@ -52,6 +53,10 @@ docker-compose up -d redis python ./check_redis_ready.py # Run aiotaskq workers in background and wait for it be ready +echo "DEBUG START" +echo "Using $(python --version) from $(which python)" +pip freeze +echo "DEBUG END" export LOG_LEVEL=${LOG_LEVEL:-INFO} aiotaskq worker sample_apps.$APP --concurrency 4 & sleep 2 @@ -63,6 +68,7 @@ python -m sample_apps.$APP.app_aiotaskq # Kill aiotaskq workers that were running in background ps | grep aiotaskq | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM + ``` ## Run a sample app with Celery @@ -71,7 +77,7 @@ Now, if you want to run the sample app with `Celery` to see how `aiotaskq` compares to it, do the following: ```bash -# ../../demo-sample-apps-simple-app-celery.sh#L1-L33 +# ../../demo-sample-apps-simple-app-celery.sh # If not already inside it, clone the aiotaskq repository and cd into it if [ "$(basename $PWD)" != "aiotaskq" ]; then @@ -84,8 +90,9 @@ fi # to choose your desired sample app. APP=simple_app -# Enter virtual env specifically for the sample apps -source ./env_activate.sh ./src/sample_apps/.venv +# Create and activate virtual env specifically for the sample apps +source ./env_create.sh ./src/sample_apps/.venv/ +source ./env_activate.sh ./src/sample_apps/.venv/ # Install sample_apps package from local file ./install_dependencies.sh ./src/sample_apps/ @@ -106,4 +113,5 @@ LOG_LEVEL=INFO python3.10 -m sample_apps.$APP.app_celery # Kill celery workers that were running in background ps | grep celery | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM + ``` From 1b277c5217ffce6bc916a40dc4b965c976085a00 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Sun, 18 Dec 2022 17:43:49 -0500 Subject: [PATCH 38/42] Fixup --- env_create.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/env_create.sh b/env_create.sh index a503fcb..61c7379 100755 --- a/env_create.sh +++ b/env_create.sh @@ -1,15 +1,14 @@ #!/bin/bash -py_version=$(python3 --version | grep -o 3.10) +venv=${1:-.venv} -if [ "$py_version" != "3.10" ]; then +python3.10 -m venv $venv +if [ "$?" != "0" ]; +then echo "Please install Python 3.10.x" exit 1 fi -venv=${1:-.venv} -python3 -m venv $venv source $venv/bin/activate - -echo "Using $(python --version)" -echo "Using $(pip --version)" +echo "Using virtual environment $venv using $(python --version)" +deactivate From 574899397e94fea2fd59407b4b44017f069efd68 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Mon, 19 Dec 2022 22:28:06 -0500 Subject: [PATCH 39/42] Debug multiple jobs --- .github/workflows/build.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3b90448..225023d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,8 +7,8 @@ on: branches: - main jobs: - run_unit_tests: - timeout-minutes: 10 + prepare: + timeout-minutes: 3 runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 @@ -22,4 +22,10 @@ jobs: - run: python ./check_redis_ready.py - run: pip freeze - run: ./env_create.sh ./src/sample_apps/.venv/ + + run_unit_tests: + needs: [prepare] + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: - run: ./test.sh From 5daf10de5b7481c2f4703ca312b5a9fe48f15cfd Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Mon, 19 Dec 2022 22:59:43 -0500 Subject: [PATCH 40/42] Revert "Debug multiple jobs" This reverts commit 574899397e94fea2fd59407b4b44017f069efd68. --- .github/workflows/build.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 225023d..3b90448 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,8 +7,8 @@ on: branches: - main jobs: - prepare: - timeout-minutes: 3 + run_unit_tests: + timeout-minutes: 10 runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 @@ -22,10 +22,4 @@ jobs: - run: python ./check_redis_ready.py - run: pip freeze - run: ./env_create.sh ./src/sample_apps/.venv/ - - run_unit_tests: - needs: [prepare] - timeout-minutes: 5 - runs-on: ubuntu-latest - steps: - run: ./test.sh From c1ff6c65643df211860c154d3b5aab698a9214de Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Mon, 19 Dec 2022 23:28:50 -0500 Subject: [PATCH 41/42] Debug check docs --- .github/workflows/build.yaml | 22 ++++++++++++++-------- demo-sample-apps-simple-app-aiotaskq.sh | 10 +++++----- demo-sample-apps-simple-app-celery.sh | 10 +++++----- src/sample_apps/simple_app/app_celery.py | 1 + src/tests/test_docs_sample_apps.py | 13 ------------- test_prepare.sh | 7 +++++++ 6 files changed, 32 insertions(+), 31 deletions(-) delete mode 100644 src/tests/test_docs_sample_apps.py create mode 100755 test_prepare.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3b90448..20100e5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,19 +7,25 @@ on: branches: - main jobs: - run_unit_tests: + tests: timeout-minutes: 10 runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - run: docker-compose -f ./docker-compose.yml up --detach redis - uses: actions/setup-python@v3 with: python-version: "3.10.x" - - run: ./env_create.sh ./src/tests/.venv/ - - run: source ./env_activate.sh ./src/tests/.venv/ - - run: ./install_dependencies.sh ./src/tests/ - - run: python ./check_redis_ready.py - - run: pip freeze - - run: ./env_create.sh ./src/sample_apps/.venv/ + - run: source test_prepare.sh - run: ./test.sh + + test_docs: + timeout-minutes: 3 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v3 + with: + python-version: "3.10.x" + - run: ./demo-sample-apps-simple-app-aiotaskq.sh + - run: ./demo-sample-apps-simple-app-celery.sh + - run: ./check-docs.sh diff --git a/demo-sample-apps-simple-app-aiotaskq.sh b/demo-sample-apps-simple-app-aiotaskq.sh index 9bd42f3..3415725 100755 --- a/demo-sample-apps-simple-app-aiotaskq.sh +++ b/demo-sample-apps-simple-app-aiotaskq.sh @@ -10,7 +10,7 @@ fi APP=simple_app # Create and activate virtual env specifically for the sample apps -source ./env_create.sh ./src/sample_apps/.venv/ +./env_create.sh ./src/sample_apps/.venv/ source ./env_activate.sh ./src/sample_apps/.venv/ # Install sample_apps package from local file @@ -29,10 +29,10 @@ export LOG_LEVEL=${LOG_LEVEL:-INFO} aiotaskq worker sample_apps.$APP --concurrency 4 & sleep 2 +# Prepare trap that kills celery workers that were running in background on script exit +trap "ps | grep aiotaskq | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM" EXIT + # Run the the sample app -python -m sample_apps.$APP.app_aiotaskq +python -m sample_apps.$APP.app_aiotaskq || exit 1 # Confirm in the logs if the app is running correctly - -# Kill aiotaskq workers that were running in background -ps | grep aiotaskq | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM diff --git a/demo-sample-apps-simple-app-celery.sh b/demo-sample-apps-simple-app-celery.sh index cdacd58..9c03806 100755 --- a/demo-sample-apps-simple-app-celery.sh +++ b/demo-sample-apps-simple-app-celery.sh @@ -10,7 +10,7 @@ fi APP=simple_app # Create and activate virtual env specifically for the sample apps -source ./env_create.sh ./src/sample_apps/.venv/ +./env_create.sh ./src/sample_apps/.venv/ source ./env_activate.sh ./src/sample_apps/.venv/ # Install sample_apps package from local file @@ -25,10 +25,10 @@ export LOG_LEVEL=${LOG_LEVEL:-INFO} celery -A sample_apps.$APP worker --concurrency 4 & sleep 2 +# Prepare trap that kills celery workers that were running in background on script exit +trap "ps | grep celery | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM" EXIT + # Run the the sample app -LOG_LEVEL=INFO python3.10 -m sample_apps.$APP.app_celery +python -m sample_apps.$APP.app_celery || exit 1 # Confirm in the logs if the app is running correctly - -# Kill celery workers that were running in background -ps | grep celery | sed 's/^[ \t]*//;s/[ \t]*$//' | cut -d ' ' -f 1 | xargs kill -TERM diff --git a/src/sample_apps/simple_app/app_celery.py b/src/sample_apps/simple_app/app_celery.py index c0000ca..6ce94e3 100644 --- a/src/sample_apps/simple_app/app_celery.py +++ b/src/sample_apps/simple_app/app_celery.py @@ -34,6 +34,7 @@ def main(): ret = apply_formula().apply_async().get() logger.info("Result: %s", ret) assert ret == 15 + assert False return ret diff --git a/src/tests/test_docs_sample_apps.py b/src/tests/test_docs_sample_apps.py deleted file mode 100644 index ee6702b..0000000 --- a/src/tests/test_docs_sample_apps.py +++ /dev/null @@ -1,13 +0,0 @@ -import subprocess - - -def test_simple_app(): - proc_aiotaskq = subprocess.run( - ["bash", "./demo-sample-apps-simple-app-aiotaskq.sh"], - check=False, - ) - proc_celery = subprocess.run( - ["bash", "./demo-sample-apps-simple-app-celery.sh"], - check=False, - ) - assert proc_aiotaskq.returncode == proc_celery.returncode == 0 diff --git a/test_prepare.sh b/test_prepare.sh new file mode 100755 index 0000000..d66752f --- /dev/null +++ b/test_prepare.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +docker-compose -f ./docker-compose.yml up --detach redis +./env_create.sh ./src/tests/.venv/ +source ./env_activate.sh ./src/tests/.venv/ +./install_dependencies.sh ./src/tests/ +python ./check_redis_ready.py From 72520a17cb66d23d2f506519b27160fbbc9d8a00 Mon Sep 17 00:00:00 2001 From: Imran Ariffin Date: Wed, 21 Dec 2022 20:52:38 -0500 Subject: [PATCH 42/42] Debug --- demo-sample-apps-simple-app-aiotaskq.sh | 2 +- src/aiotaskq/__main__.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/demo-sample-apps-simple-app-aiotaskq.sh b/demo-sample-apps-simple-app-aiotaskq.sh index 3415725..7d2f02c 100755 --- a/demo-sample-apps-simple-app-aiotaskq.sh +++ b/demo-sample-apps-simple-app-aiotaskq.sh @@ -26,7 +26,7 @@ echo "Using $(python --version) from $(which python)" pip freeze echo "DEBUG END" export LOG_LEVEL=${LOG_LEVEL:-INFO} -aiotaskq worker sample_apps.$APP --concurrency 4 & +aiotaskq worker sample_apps.$APP.aiotaskq --concurrency 4 & sleep 2 # Prepare trap that kills celery workers that were running in background on script exit diff --git a/src/aiotaskq/__main__.py b/src/aiotaskq/__main__.py index 50f6987..bfa4699 100755 --- a/src/aiotaskq/__main__.py +++ b/src/aiotaskq/__main__.py @@ -2,6 +2,8 @@ #!/usr/bin/env python +import logging +import os import typing as t import typer @@ -12,6 +14,9 @@ cli = typer.Typer() +log_level = os.environ.get("LOG_LEVEL", "INFO") +logging.basicConfig(level=log_level) + def _version_callback(value: bool): """Ensure print the current version and exit cli if `--version` param is provided."""