diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml new file mode 100644 index 0000000..d8498e6 --- /dev/null +++ b/.github/workflows/documentation.yaml @@ -0,0 +1,18 @@ +name: Deploy Sphinx documentation to Pages + +on: + push: + branches: [main] # branch to trigger deployment + +jobs: + pages: + runs-on: ubuntu-20.04 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + pages: write + id-token: write + steps: + - id: deployment + uses: sphinx-notes/pages@v3 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eeb8a6e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +**/__pycache__ diff --git a/README.md b/README.md index 85a0bca..f455704 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,3 @@ You can install YDL from PyPI: ``` python -m pip install ydl-ipc ``` - -## Documentation - -Check out the tutorial in the docs folder! diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..69fa449 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +_build/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..dab47e5 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,14 @@ +# Docs + +## Quickstart + +To run build/run the docs locally, do the following in this `docs` folder: + + - Install docs dependencies: `pip install -r requirements.txt` + - run `make html` to generate html files + - in `_build/html` folder, run `python3 -m http.server` + + +## How does this work? + +The documentation is built using Sphinx, using the Read The Docs theme, and using the Sphinx autodoc extension for the API page. It's deployed to github pages via a [github action](../.github/workflows/documentation.yaml). diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..ec3296f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,36 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +import sys +import os +sys.path.append(os.path.abspath('..')) # add source for autodocs + +project = 'YDL' +copyright = '2022, Pioneers in Engineering' +author = 'Pioneers in Engineering' +release = '0.2.0' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + + + +extensions = [ + "sphinx.ext.autodoc" +] + +templates_path = [] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = [] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..cc4d7da --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,24 @@ +.. ydl documentation master file, created by + sphinx-quickstart on Wed Jun 28 20:01:50 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +YDL +=============================== + +YDL is a simple inter-process communication framework. It is cross-platform and built on TCP sockets. + +The name stands for Yheavyweight Dcommunications and Lmarshalling, with the acronym being pronounced "yodel". + + + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + installation.rst + tutorial.rst + reference.rst + technicals.rst + + diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..d77c2a7 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,9 @@ +Installation +============= + + +Install YDL with pip: + +.. code-block:: bash + + python3 -m pip install --upgrade ydl-ipc diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..6eff0ca --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,26 @@ +.. py:module:: ydl +.. py:currentmodule:: ydl + +API Reference +================= + +.. py:attribute:: DEFAULT_HOST + :type: str + + The default host that the server and client bind to. + +.. py:attribute:: DEFAULT_PORT + :type: int + + The default port that the server and client bind to. + +.. autoclass:: Client + :members: + +.. autoclass:: Handler + :members: + +.. autofunction:: header + +.. autofunction:: run_server + diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..0d4588c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +# requirements for building docs +sphinx +sphinx_rtd_theme diff --git a/docs/technicals.rst b/docs/technicals.rst new file mode 100644 index 0000000..c614c6a --- /dev/null +++ b/docs/technicals.rst @@ -0,0 +1,11 @@ +Random Notes +========================= + + +Note that YDL operates on a many-to-many messaging model, so you can have several processes listen on the same channel if needed. This might be useful for logging; you can have a logging process listen to all your channels, without changing the behavior of your application at all. + +A client may listen to any number of channels, as long as they're all passed as arguments to the constructor. + +Clients will connect to the server in the ``Client()`` constructor; note that this will block if the server isn't available. Both ``send()`` and ``receive()`` will block and try to reconnect if the connection is lost. + +If the server goes down, it may simply be restarted without too much chaos. Some messages may be lost if they were sent just before the server went down or just after it comes back up. This is somewhat unavoidable, so if you need to guarantee that a message was sent, you should implement confirmation messages. Actually, if you need that kind of guarantee, you should just use an industrial-strength solution like RabbitMQ or Apache Kafka. diff --git a/docs/tutorial.md b/docs/tutorial.md deleted file mode 100644 index 1688482..0000000 --- a/docs/tutorial.md +++ /dev/null @@ -1,200 +0,0 @@ -# An Introduction to YDL - -## What is YDL? - -YDL (pronounced "yodel") is a simple inter-process communication framework. Processes that want to communicate have _clients_ (usually one per process), and they can talk to each other through a common _server_. The server may be run on its own, or as part of one of the processes. - -Within each server are several _channels_. Each channel is a unique string identifier, which determines where messages are forwarded. Any client may send a message across any channel, and it will be received by all clients listening on that channel. If there are no clients listening to a given channel, the message is simply discarded. - -## A minimal example - -(Prerequisites: having python3 and ydl installed) - -The smallest useful example is one Python process sending a message to another Python process. For this, you will need to open three terminal windows. Execute the following command in terminal 1: -``` -$ python3 -m ydl -``` -This command starts the YDL server, which the two other processes will connect to. It's not important to run this command first; if you start the other two processes first, they would simply wait for the server to become availible until you ran this command. - -Execute the following in terminal 2: -``` -$ python3 ->>> import ydl ->>> yc = ydl.YDLClient("cheese") ->>> yc.receive() -``` -You'll notice that after executing the last line, the interpreter will seem to freeze. This is good! The call to `yc.receive()` will wait for the next message across any of the channels that `yc` is listening on. Currently, `yc` is listening to the channel `"cheese"`. - -Execute the following in terminal 3: -``` -$ python3 ->>> import ydl ->>> yc = ydl.YDLClient() ->>> yc.send(("cheese", 1, 2, 3, "cool")) -``` -After executing `yc.send()`, you should notice the full tuple pop up in terminal 2. Congratulations, you've sent your first message! - -You should try some other `send()` calls; it can send any tuple whose elements are json serializable. The first element of the tuple must be the channel you want to send to; in this case, `"cheese"`. - -After calling `send()` a bunch of times, you can call `receive()` in terminal 2 the same number of times to retrieve the messages. - -## A less minimal example - -The minimal example involves two processes talking to each other on the same computer; however, YDL can do much more than that. In this less minimal example, we'll demonstrate: - - running the YDL server as a thread on one of the processes - - communication between two processes on different computers - - two-way communication - -For this example, you will need two terminal windows on seperate computers. You may run both terminals on the same computer, but it's more interesting if you run them on two seperate computers on the same local network. - -In terminal 1, execute: -``` -$ python3 ->>> import threading ->>> import ydl ->>> def run_server_locally(): -... ydl.run_ydl_server("0.0.0.0", 5001) -... ->>> threading.Thread(target=run_server_locally).start() ->>> yc = ydl.YDLClient("potato", "banana") ->>> while True: -... m = yc.receive() -... yc.send(("cheese",) + m[1:]) -``` -There are a few interesting things here. First, instead of running the YDL server as its own process, it's being run as a thread on this process. Secondly, the server is listening on `0.0.0.0`, rather than the default `127.0.0.1` (this will allow it to accept connections for the local area network). Finally, the while loop means that the `yc` client will listen for any messages to `"potato"` or `"banana"` and forward them to `"cheese"`. - -Now, determine the local IP address of the first computer (this can be done with `ip addr` on Linux, or by looking at the network preferences on Mac). In terminal 2, execute -``` -$ python3 ->>> import ydl ->>> yc = ydl.YDLClient("cheese", socket_address=("COMPUTER_1_IP_HERE", 5001)) ->>> yc.send(("potato", 1234)) ->>> yc.receive() -``` -(make sure to replace `"COMPUTER_1_IP_HERE"` with the IP address of your first computer, or `"127.0.0.1"` if you're running both terminal windows on the same computer) - -You should see the message `('cheese', 1234)` received back. If so, congratulations! You've successfully had two processes communicate across a network. - -## Structured Communication - -By default, messages are very permissive - you can pretty much send any tuple that begins with a channel name. However, such flexible comminication can become unwieldy for larger projects. - -One common use case of YDL is for remote procedure calls; basically, we want to invoke some function on the receiving process. For example, we may have two processes that do something like this (make sure to run `python3 -m ydl` in a 3rd terminal if you want to run this demo): - -Process 1: -```python -import ydl -yc = ydl.YDLClient("interactive") -while True: - num = int(input("Enter an integer: ")) - op = input("Enter i to increment, or d to double: ") - if op == "i": - yc.send(("calculator", "i", {"num": num})) - elif op == "d": - yc.send(("calculator", "d", {"num": num})) - else: - print("unsupported operation") - continue - print("result: ", yc.receive()[1]) -``` -Process 2: -```python -import ydl - -def increment(num): - return num + 1 - -def double(num): - return num * 2 - -yc = ydl.YDLClient("calculator") -fn_mapping = {"i": increment, "d": double} -while True: - _, op, data = yc.receive() - yc.send(("interactive", fn_mapping[op](**data))) -``` -Here, we have process 1 doing remote procedure calls supported by process 2. However, there are some clunky bits here: - - function names and arguments are identified by strings, which is just inviting misspellings - - whenever we want to send a message from process 1, we have to remember all the arguments - - no autocompletion :( - - no type checking - -All of these problems can be solved through the use of _header functions_, which are a mechanism for creating structured messages. Let's modify the previous example: - -First, a new file called `shared.py`: -```python -import ydl - -calc_channel = "calculator" -int_channel = "interactive" - -@ydl.header(calc_channel, "i") -def increment_message(num: int): - pass - -@ydl.header(calc_channel, "d") -def double_message(num: int): - pass - -@ydl.header(int_channel, "result") -def result_message(num: int): - pass -``` -Process 1: -```python -import ydl -from shared import * -yc = ydl.YDLClient(int_channel) -while True: - num = int(input("Enter an integer: ")) - op = input("Enter i to increment, or d to double: ") - if op == "i": - yc.send(increment_message(num=num)) - elif op == "d": - yc.send(double_message(num=num)) - else: - print("unsupported operation") - continue - print("result: ", yc.receive()[2]['num']) -``` -Process 2: -```python -import ydl -from shared import * - -def increment(num): - return num + 1 - -def double(num): - return num * 2 - -yc = ydl.YDLClient(calc_channel) -fn_mapping = { - increment_message.name: increment, - double_message.name: double -} -while True: - _, op, data = yc.receive() - yc.send(result_message(num=fn_mapping[op](**data))) -``` - -The annotion `@ydl.header` automatically replaces the function with one that will construct a message. The new function will do the following: - - make sure all arguments are keyword, not positional (makes contents of messages more explicit) - - typecheck all the arguments - - call the original function, which has the opportunity to raise errors. For example, the original function might do bounds checks on the arguments. - - finally, return the tuple (target_channel, header, {args}). Usually "header" corresponds to a function name. - -The new function also gets new properties/variables: `fn.target` and `fn.header`, which correspond to the target channel and the header respectively. This allows the receiving end to avoid hardcoding the names. - -## Technical Behavior - -Note that YDL operates on a many-to-many messaging model, so you can have several processes listen on the same channel if needed. This might be useful for logging; you can have a logging process listen to all your channels, without changing the behavior of your application at all. - -A client may listen to any number of channels, as long as they're all passed as arguments to the constructor. - -Clients will connect to the server in the `YDLClient()` constructor; note that this will block if the server isn't availible. Both `send()` and `receive()` will block and try to reconnect if connection is lost. - -If the server goes down, it may simply be restarted without too much chaos. Some messages may be lost if they were sent just before the server went down, or just after it comes back up. This is somewhat unavoidable, so if you need to guarentee that a message was sent, you should implement confirmation messages. - - - diff --git a/docs/tutorial.rst b/docs/tutorial.rst new file mode 100644 index 0000000..f60b6f1 --- /dev/null +++ b/docs/tutorial.rst @@ -0,0 +1,205 @@ +Tutorial +====================== + +What is YDL? +------------- + +YDL (pronounced "yodel") is a simple inter-process communication framework. Processes that want to communicate have *clients* (usually one per process), and they can talk to each other through a common *server*. The server may be run on its own, or as part of one of the processes. + +Within each server are several *channels*. Each channel is a unique string identifier, which determines where messages are forwarded. Any client may send a message across any channel, and it will be received by all clients listening on that channel. If there are no clients listening to a given channel, the message is simply discarded. + +A minimal example +----------------- + +(Prerequisites: having python3 and ydl installed) + +The smallest useful example is one Python process sending a message to another Python process. For this, you will need to open three terminal windows. Execute the following command in terminal 1: + +.. code-block:: bash + + $ python3 -m ydl + +This command starts the YDL server, which the two other processes will connect to. It's not important to run this command first; if you start the other two processes first, they would simply wait for the server to become available until you ran this command. + +Execute the following in terminal 2: + +.. code-block:: bash + + $ python3 + >>> import ydl + >>> yc = ydl.Client("cheese") + >>> yc.receive() + +You'll notice that after executing the last line, the interpreter will seem to freeze. This is good! The call to ``yc.receive()`` will wait for the next message across any of the channels that ``yc`` is listening on. Currently, ``yc`` is listening to the channel "cheese". + +Execute the following in terminal 3: + +.. code-block:: bash + + $ python3 + >>> import ydl + >>> yc = ydl.Client() + >>> yc.send(("cheese", 1, 2, 3, "cool")) + +After executing ``yc.send()``, you should notice the full tuple pop up in terminal 2. Congratulations, you've sent your first message! + +You should try some other ``send()`` calls; it can send any tuple whose elements are JSON serializable. The first element of the tuple must be the channel you want to send to; in this case, "cheese". + +After calling ``send()`` a bunch of times, you can call ``receive()`` in terminal 2 the same number of times to retrieve the messages. + +A less minimal example +---------------------- + +The minimal example involves two processes talking to each other on the same computer; however, YDL can do much more than that. In this less minimal example, we'll demonstrate: + +- running the YDL server as a thread on one of the processes +- communication between two processes on different computers +- two-way communication + +For this example, you will need two terminal windows on separate computers. You may run both terminals on the same computer, but it's more interesting if you run them on two separate computers on the same local network. + +In terminal 1, execute: + +.. code-block:: bash + + $ python3 + >>> import threading + >>> import ydl + >>> threading.Thread(target=ydl.run_server, args=("0.0.0.0", 5001)).start() + >>> yc = ydl.Client("potato", "banana") + >>> while True: + ... m = yc.receive() + ... yc.send(("cheese",) + m) + +There are a few interesting things here. First, instead of running the YDL server as its own process, it's being run as a thread on this process. Secondly, the server is listening on ``0.0.0.0``, rather than the default ``127.0.0.1`` (this will allow it to accept connections for the local area network). Finally, the while loop means that the ``yc`` client will listen for any messages to "potato" or "banana" and forward them to "cheese". + +Now, determine the local IP address of the first computer (this can be done with ``ip addr`` on Linux, or by looking at the network preferences on Mac). In terminal 2, execute: + +.. code-block:: bash + + $ python3 + >>> import ydl + >>> yc = ydl.Client("cheese", host="COMPUTER_1_IP_HERE", port=5001) + >>> yc.send(("potato", 1234)) + >>> yc.receive() + +(make sure to replace "COMPUTER_1_IP_HERE" with the IP address of your first computer, or "127.0.0.1" if you're running both terminal windows on the same computer) + +You should see the message ``('cheese', 'potato', 1234)`` received back. If so, congratulations! You've successfully had two processes communicate across a network. + +Structured Communication +------------------------ + +By default, messages are very permissive - you can pretty much send any tuple that begins with a channel name. However, such flexible communication can become unwieldy for larger projects. + +One common use case of YDL is for remote procedure calls; basically, we want to invoke some function on the receiving process. For example, we may have two processes that do something like this (make sure to run ``python3 -m ydl`` in a 3rd terminal if you want to run this demo): + +Process 1: + +.. code-block:: python + + import ydl + yc = ydl.Client("interactive") + while True: + num = int(input("Enter an integer: ")) + op = input("Enter i to increment, or d to double: ") + if op == "i": + yc.send(("calculator", "i", {"num": num})) + elif op == "d": + yc.send(("calculator", "d", {"num": num})) + else: + print("unsupported operation") + continue + print("result: ", yc.receive()[1]) + +Process 2: + +.. code-block:: python + + import ydl + + def increment(num): + return num + 1 + + def double(num): + return num * 2 + + yc = ydl.Client("calculator") + fn_mapping = {"i": increment, "d": double} + while True: + _, op, data = yc.receive() + yc.send(("interactive", fn_mapping[op](**data))) + +Here, we have process 1 doing remote procedure calls supported by process 2. However, there are some clunky bits here: + +- function names and arguments are identified by strings, which is just inviting misspellings +- whenever we want to send a message from process 1, we have to remember all the arguments +- no autocompletion :( + +All of these problems can be solved through the use of *header functions*, which are a mechanism for creating structured messages. Let's modify the previous example: + +First, a new file called `shared.py`: + +.. code-block:: python + + import ydl + + calc_channel = "calculator" + int_channel = "interactive" + + @ydl.header(calc_channel, "i") + def increment_message(num: int): + pass + + @ydl.header(calc_channel, "d") + def double_message(num: int): + pass + + @ydl.header(int_channel, "result") + def result_message(num: int): + pass + +Process 1: + +.. code-block:: python + + import ydl + from shared import * + + yc = ydl.Client(int_channel) + while True: + num = int(input("Enter an integer: ")) + op = input("Enter i to increment, or d to double: ") + if op == "i": + yc.send(increment_message(num)) + elif op == "d": + yc.send(double_message(num)) + else: + print("unsupported operation") + continue + print("result: ", yc.receive()[2]['num']) + +Process 2: + +.. code-block:: python + + import ydl + from shared import * + + yc = ydl.Client(calc_channel) + yh = ydl.Handler() + + @yh.on(increment_message) + def increment(num): + yc.send(result_message(num + 1)) + + @yh.on(double_message) + def double(num): + yc.send(result_message(num * 2)) + + while True: + yh.handle(yc.receive()) + +The annotation ``@ydl.header`` automatically replaces the function with one that will construct a message. The new function will also call the original function, which has the opportunity to raise errors (for example, type checking). + +The annotation ``@yh.on`` adds a function to the given ``Handler`` object, so that the function will be called whenever that type of message is received. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..de97c9e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "ydl-ipc" +packages = [ + { include = "ydl" } +] +version = "0.2.0" +description = "Simple inter-process communication" +authors = ["Pioneers in Engineering "] +readme = "README.md" +license = "MIT" +classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", +] diff --git a/setup.py b/setup.py deleted file mode 100644 index 7f65f1e..0000000 --- a/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Setup script for YDL""" - -from setuptools import setup - - -setup( - name="ydl-ipc", - packages=["ydl"], - version="0.1.0", - description="Simple inter-process communication", - long_description="Read the README at https://github.com/pioneers/ydl", - url="https://github.com/pioneers/ydl", - license="MIT", - classifiers=[ - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - ], - install_requires=["typeguard"], -) diff --git a/ydl/__init__.py b/ydl/__init__.py index e27a385..d1ee156 100644 --- a/ydl/__init__.py +++ b/ydl/__init__.py @@ -3,20 +3,20 @@ Import the ydl module to initialize a YDL client or server: Process 0: - >>> from ydl import run_ydl_server - >>> run_ydl_server() + >>> from ydl import run_server + >>> run_server() Process 1: - >>> from ydl import YDLClient - >>> yc = YDLClient() + >>> from ydl import Client + >>> yc = Client() >>> yc.send(("process 2", "stuff")) Process 2: - >>> from ydl import YDLClient - >>> yc = YDLClient("process 2") + >>> from ydl import Client + >>> yc = Client("process 2") >>> yc.receive() ('process 2', 'stuff') """ -from ._core import DEFAULT_YDL_ADDR, YDLClient, run_ydl_server -from ._header import header \ No newline at end of file +from ._core import DEFAULT_HOST, DEFAULT_PORT, Client, run_server +from ._header import header, Handler diff --git a/ydl/__main__.py b/ydl/__main__.py index 0a47cd1..55bcac4 100644 --- a/ydl/__main__.py +++ b/ydl/__main__.py @@ -5,15 +5,15 @@ import argparse import signal import sys -from ._core import run_ydl_server +from ._core import run_server, DEFAULT_HOST, DEFAULT_PORT if __name__ == "__main__": parser = argparse.ArgumentParser(prog='python3 -m ydl', description=__doc__) parser.add_argument("-s", "--silent", action='store_true', help="do not print client connections") - parser.add_argument("-a", "--address", default=None, type=str, - help="run server on specified ip address, such as 0.0.0.0") - parser.add_argument("-p", "--port", default=None, type=int, + parser.add_argument("-a", "--address", default=DEFAULT_HOST, type=str, + help="run server on specified host address, such as 0.0.0.0") + parser.add_argument("-p", "--port", default=DEFAULT_PORT, type=int, help="run server on specified port") args = parser.parse_args() @@ -25,4 +25,4 @@ def sigint_handler(_signum, _frame): sys.exit(0) signal.signal(signal.SIGINT, sigint_handler) - run_ydl_server(args.address, args.port, verbose) + run_server(args.address, args.port, verbose) diff --git a/ydl/_core.py b/ydl/_core.py index 27869bb..5b69e19 100644 --- a/ydl/_core.py +++ b/ydl/_core.py @@ -11,20 +11,20 @@ # A network of clients should communicate through one server on a designated address. # if all clients are on one computer, 127.0.0.1 works; if distributed across a local # network, use 0.0.0.0 instead. -DEFAULT_YDL_ADDR = ('127.0.0.1', 5001) # doesn't need to be available on network +DEFAULT_HOST = '127.0.0.1' # doesn't need to be available on network +DEFAULT_PORT = 5001 - -class YDLClient(): +class Client(): ''' A client to the YDL network. Listens to a set of channels, and may send messages to any channel. Will automatically try to connect to YDL network. ''' - def __init__(self, *receive_channels: str, socket_address: Tuple[str, int] = DEFAULT_YDL_ADDR): + def __init__(self, *receive_channels: str, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT): ''' Waits for connection to open, then subscribes to given receive_channels ''' self._receive_channels = receive_channels - self._socket_address = socket_address + self._socket_address = (host, port) self._lock = threading.Lock() # protects all local variables self._conn = None # set in _new_connection() self._selobj = None # set in _new_connection() @@ -33,10 +33,11 @@ def __init__(self, *receive_channels: str, socket_address: Tuple[str, int] = DEF def send(self, message: Tuple): ''' - The input message is a tuple of (target_channel, stuff...) - target_channel: str, which channel you want to send to - stuff: any other stuff you want to send. Must be json serializable. - May block if disconnected and waiting for a new connection. + The input message is a tuple of `(target_channel, stuff...)`. + `target_channel` is a string which identifies the channel you want to + send to. `stuff` can be any other stuff you want to send. `stuff` must + be json serializable. This method may block if disconnected and + waiting for a new connection. ''' target_channel = message[0] json_str = json.dumps(message[1:]) @@ -181,22 +182,17 @@ def read(sel, subscriptions, conn, obj, verbose): for chan in subscriptions[target_channel]: send_message(chan, target_channel, message) -def run_ydl_server(address=None, port=None, verbose=False): +def run_server(host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, verbose: bool = False): ''' - (server method - internal use only) Runs the YDL server that processes will use to communicate with each other ''' - if address is None: - address = DEFAULT_YDL_ADDR[0] - if port is None: - port = DEFAULT_YDL_ADDR[1] if verbose: - print("Starting YDL server at address:", (address, port)) + print("Starting YDL server at address:", (host, port)) subscriptions = {} # a mapping of target names -> list of socket objects sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((address, port)) + sock.bind((host, port)) sock.listen() sock.setblocking(False) sel = selectors.DefaultSelector() diff --git a/ydl/_header.py b/ydl/_header.py index d1c76f9..db1779b 100644 --- a/ydl/_header.py +++ b/ydl/_header.py @@ -1,100 +1,79 @@ -from typeguard import check_type -# global list of header names, used to check for collisions -global_header_names = [] -def header(target, name): +import inspect +from typing import Tuple + +def header(channel, name): """ - The decorator wrapper function that sets the target and name for the - decorator environment. + Decorator that turns a regular function into a "header function". The + header function calls the original function (so it can do input validation), + then returns a message in this format: + (channel, name, args) + where args is a dictionary of the arguments """ - # make sure there are no name collisions - if target+name in global_header_names: - raise ValueError(f"header name collision: {target+name}") - global_header_names.append(target+name) def make_header(func): - """ - The inside decorator function that will get called on the decorated - function. - Returns a HeaderPrimitive object with the appropriate target, name, and - typing_function. - """ - # this is magic - # purposefully confusing pylance so it will give up and - # show the header with the signature of the original function - # it is synonymous with `return HeaderPrimitive(target, name, func)` - return [HeaderPrimitive(target, name, func),0][0] + parameters = inspect.signature(func).parameters + arg_names = list(parameters.keys()) + default_params = {name: p.default for name, p in parameters.items()} + + def header_func(*args, **kwargs): + func(*args, **kwargs) + args_dict = default_params.copy() + args_dict.update(zip(arg_names, args)) + args_dict.update(kwargs) + return (channel, name, args_dict) + + header_func.ydl_channel = channel + header_func.ydl_name = name + header_func.ydl_arg_names = arg_names + return header_func return make_header -class HeaderPrimitive(): + + + +class Handler(): """ - The header class, which is used to type check and return formatted header - tuples. - This class is callable, and will type check the passed in keyword arguments - and return a tuple of the target, the name, and the kwargs. This class - should be created by decorating a type anotated function with the @header - decorator. Any untyped variables will not be type checked. - This class is also comparable to other header classes or to strings, and will - be equal if the name field is equal, or if the name is equal to the string. - This class is also hashable, so it may be used as a key in a dictionary. - This class is immutable as well. - This docstring will be replaced by the typing_function's docstring. + A handler object is meant to store a bunch of functions, + then call the corresponding function whenever a header is received """ - def __init__(self, target, name, typing_function): - """ - The initializer for the HeaderPrimitive class. Copies the doc string from - the typing function and sets the target, name, and typing function. - """ - self.__doc__ = typing_function.__doc__ - self.target = target - self.name = name - self.typing_function = typing_function - def __call__(self, *args, **kwargs): - """ - The function that is called when an instance of HeaderPrimitive is called. - Ensures that no positional args are passed in. - Ensures that all the keyword args that are passed in are typed the same - as the annotations. - Calls the typing function on the keyword arguments to ensure that the - function signature is satisfied and that the rules defined in the - typing_function are followed. - Returns a tuple of the target, the name, and the kwargs. - """ - # check for positional args and raise an exception - if len(args) > 0: - raise TypeError(f"{self.typing_function.__name__}" - + " requires keyword arguments, but positional arguments were given") - # check the typing for each arg in kwargs against the annotation. - annots = self.typing_function.__annotations__ - for arg, value in kwargs.items(): - # skips unannotated arguments, typechecks annotated arguments. - # skips nonetype args, those will get handled later if they are an issue. - if arg in annots and value is not None: - check_type(f"{self.typing_function.__name__}({arg})", value, annots[arg]) - # call the typing_function and ignore any return value. - # the typing_function must raise an error to have an effect. - self.typing_function(**kwargs) - # return a tuple of the target, the name, and the kwargs. - return (self.target, self.name, kwargs) - def __eq__(self, other): - """ - Do not allow ==. - """ - raise NotImplementedError - def __hash__(self) -> int: - """ - Do not allow hashing. - """ - raise NotImplementedError - def __str__(self): - """ - Return the name for str(). - """ - return self.name - def __setarr__(self, *_): - """ - Do not allow assignment to members of this class in order to make it immutable - """ - raise NotImplementedError - def __delattr__(self, *_): - """ - Do not allow deletion of members of this class in order to make it immutable - """ - raise NotImplementedError + def __init__(self): + self.mapping = {} + + def on(self, header_fn): + """ + This decorator annotates a function that the handler should call whenever + the given header is received. The original function is returned + by the decorator, so a function can be annotated multiple times. + """ + def add_function(handling_fn): + assert header_fn.ydl_name not in self.mapping, "duplicate header" + + header_params = header_fn.ydl_arg_names + handle_params = list(inspect.signature(handling_fn).parameters.keys()) + assert header_params == handle_params, "Header has params " + \ + f"{header_params} but handler has params {handle_params}" + + self.mapping[header_fn.ydl_name] = handling_fn + return handling_fn + + return add_function + + def can_handle(self, message: Tuple) -> bool: + """ + Returns True if the message is well-formed, and there exists a + corresponding function for the handler to call. + """ + return len(message) == 3 and \ + isinstance(message[1], str) and \ + isinstance(message[2], dict) and \ + message[1] in self.mapping + + def handle(self, message: Tuple) -> Tuple: + """ + If the message is well-formed and an applicable function exists for the + given message, calls that function and returns a single-element tuple + `(result,)`, where `result` is the return value of the function. + Otherwise, returns an empty tuple. Note that `(result,)` is truthy and + an empty tuple is falsy. + """ + if self.can_handle(message): + return (self.mapping.get(message[1])(**message[2]),) + return ()