From 7089124f989d5fd8e58dc3c334f5db70a8843dc6 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Tue, 13 Apr 2021 15:10:41 +0200 Subject: [PATCH 01/43] Added dask distributed to required packages --- notebooks/workers.ipynb | 45 +++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 2 files changed, 46 insertions(+) create mode 100644 notebooks/workers.ipynb diff --git a/notebooks/workers.ipynb b/notebooks/workers.ipynb new file mode 100644 index 0000000..dd55d94 --- /dev/null +++ b/notebooks/workers.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import dask.distributed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/setup.py b/setup.py index 6273fe3..ef6acfe 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ install_requires=[ 'cython', 'dask', + 'dask[distributed]' 'dask[dataframe]', 'pyarrow', 'pandas', From 916412d813b1fad92c3889a3c4300fd65e098f08 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Tue, 13 Apr 2021 15:11:26 +0200 Subject: [PATCH 02/43] Fixed syntax error in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ef6acfe..27a1d2a 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ install_requires=[ 'cython', 'dask', - 'dask[distributed]' + 'dask[distributed]', 'dask[dataframe]', 'pyarrow', 'pandas', From eb33ed6006bbf5c1204691f4be9f27354374aeeb Mon Sep 17 00:00:00 2001 From: bobluppes Date: Fri, 16 Apr 2021 16:53:49 +0200 Subject: [PATCH 03/43] Added jupyter server proxy to required packages --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 27a1d2a..3e881c8 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ 'dask', 'dask[distributed]', 'dask[dataframe]', + 'jupyter-server-proxy', 'pyarrow', 'pandas', 'xeger', From ffdb4533662f564060af5a2eeccf1e2bdefbebc2 Mon Sep 17 00:00:00 2001 From: bobluppes Date: Fri, 16 Apr 2021 16:55:29 +0200 Subject: [PATCH 04/43] Added jupyter notebooks for scheduler and worker plugins --- notebooks/scheduler-plugin.ipynb | 213 +++++++++++++++++++++++++++++++ notebooks/worker-plugin.ipynb | 165 ++++++++++++++++++++++++ 2 files changed, 378 insertions(+) create mode 100644 notebooks/scheduler-plugin.ipynb create mode 100644 notebooks/worker-plugin.ipynb diff --git a/notebooks/scheduler-plugin.ipynb b/notebooks/scheduler-plugin.ipynb new file mode 100644 index 0000000..ac70b3b --- /dev/null +++ b/notebooks/scheduler-plugin.ipynb @@ -0,0 +1,213 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import pickle\n", + "from dask.distributed import Client, Scheduler, SchedulerPlugin" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Close the previous scheduler and all associated workers if present\n", + "try:\n", + " client.shutdown()\n", + " client.close()\n", + "except:\n", + " print('Could not shut down old client, was there any to begin with?')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Start a client in local cluster mode\n", + "client = Client()\n", + "print('Dashboard available at', client.dashboard_link)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Scheduler plugin that accelerates the\n", + "# task graph using dask-accelerated\n", + "class SchedulerOptimizer(SchedulerPlugin):\n", + "\n", + " # Add the scheduler instance to the plugin as an attribute\n", + " # such that we can access the underlying task state\n", + " def __init__(self, scheduler):\n", + " self.scheduler = scheduler\n", + "\n", + " def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs):\n", + " for key in keys:\n", + " task_state = scheduler.tasks[key]\n", + " scheduler.logg(task_state)\n", + " rs_func = pickle.loads(task_state.run_spec['function'])\n", + " rs_arg = pickle.loads(task_state.run_spec['args'])\n", + " scheduler.logg(rs_func, rs_arg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "scheduler = client.cluster.scheduler" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# The client dashboard is able to display the info logs\n", + "# from the scheduler. Unfortunately, the scheduler does not\n", + "# have a method to log data, so we monkey patch the scheduler class\n", + "import logging\n", + "logger = logging.getLogger(\"distributed.scheduler\")\n", + "\n", + "# The dashboard only exposes info logs, so\n", + "# logger.info is used instead of logger.debug\n", + "def log_method(self, *msgs):\n", + " logstring = ''\n", + " for msg in msgs:\n", + " logstring += str(msg) + '\\t'\n", + "\n", + " logger.info(logstring)\n", + "\n", + "# The monkey patched method\n", + "Scheduler.logg = log_method\n", + "\n", + "# TODO: remove\n", + "scheduler.logg('logger test')\n", + "scheduler.logg('logger test', 'multiple inputs')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Register the plugin with the scheduler\n", + "plugin = SchedulerOptimizer(scheduler)\n", + "scheduler.add_plugin(plugin)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Define a simple function and\n", + "# submit a future on the client\n", + "\n", + "# Increment integer values by 1\n", + "def inc(x):\n", + " return x + 1\n", + "\n", + "# Y holds the future to the result\n", + "y = client.submit(inc, 10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Print the result of the future when it is ready\n", + "y.result()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Restart the client in case something went wrong\n", + "# By default this doesnt run so the entire notebook\n", + "# can be executed without restarting the client\n", + "if False:\n", + " client.restart()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/worker-plugin.ipynb b/notebooks/worker-plugin.ipynb new file mode 100644 index 0000000..a0e4fa9 --- /dev/null +++ b/notebooks/worker-plugin.ipynb @@ -0,0 +1,165 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dask.distributed import Client, WorkerPlugin" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Close the previous scheduler and all associated workers if present\n", + "try:\n", + " client.shutdown()\n", + " client.close()\n", + "except:\n", + " print('Could not shut down old client, was there any to begin with?')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# First start a scheduler and worker from cli:\n", + "# $ dask-scheduler\n", + "# $ dask-worker tcp://xxx.xx.xxx.xxx:pppp --nthreads 1 --memory-limit 0 --no-nanny\n", + "\n", + "# Start a client and connect it to the existing scheduler\n", + "client = Client('tcp://145.94.225.114:8786')\n", + "print('Dashboard available at', client.dashboard_link)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Worker plugin that accelerates the incoming part of the\n", + "# task graph using dask-accelerated\n", + "class WorkerOptimizer(WorkerPlugin):\n", + "\n", + " def setup(self, worker):\n", + " print('WorkerOptimizer plugin registered')\n", + "\n", + " def transition(self, key, start, finish, **kwargs):\n", + " print(key, '\\t', start, '\\t', finish)\n", + " print(kwargs, '\\n')\n", + "\n", + " def release_key(self, key, state, cause, reason, report):\n", + " print(key, '\\t', state, '\\t', cause, '\\t', reason, '\\t', report)\n", + "\n", + " def release_dep(self, dep, state, report):\n", + " print(dep, '\\t', state, '\\t', report)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Register the plugin with all current and future workers\n", + "client.register_worker_plugin(WorkerOptimizer)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Define a simple function and\n", + "# submit a future on the client\n", + "\n", + "# Increment integer values by 1\n", + "def inc(x):\n", + " return x + 1\n", + "\n", + "# Y holds the future to the result\n", + "y = client.submit(inc, 10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Print the result of the future when it is ready\n", + "y.result()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Restart the client in case something went wrong\n", + "# By default this doesnt run so the entire notebook\n", + "# can be executed without restarting the client\n", + "if False:\n", + " client.restart()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 245ee283fca77b96b20db55be72a6ea608b5cfbc Mon Sep 17 00:00:00 2001 From: bobluppes Date: Fri, 16 Apr 2021 16:56:30 +0200 Subject: [PATCH 05/43] Removed depricated workers.ipynb --- notebooks/workers.ipynb | 45 ----------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 notebooks/workers.ipynb diff --git a/notebooks/workers.ipynb b/notebooks/workers.ipynb deleted file mode 100644 index dd55d94..0000000 --- a/notebooks/workers.ipynb +++ /dev/null @@ -1,45 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import dask.distributed" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} From 89ad246cb433a93f9a5727b2060c304142d4387d Mon Sep 17 00:00:00 2001 From: bobluppes Date: Fri, 16 Apr 2021 17:00:36 +0200 Subject: [PATCH 06/43] Added dask distributed worker space to .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 70d1329..a8db7a0 100644 --- a/.gitignore +++ b/.gitignore @@ -150,4 +150,7 @@ cython_debug/ *.pdf # Profiler results -profiler/*.txt \ No newline at end of file +profiler/*.txt + +# Dask distributed +/dask-worker-space \ No newline at end of file From 2da94fe13a9e41a2aad480bc869f9a928ae07ff9 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Fri, 23 Apr 2021 14:43:57 +0200 Subject: [PATCH 07/43] Added some cells for testing the connection between a scheduler and worker node --- notebooks/accelerated-worker.ipynb | 308 +++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 notebooks/accelerated-worker.ipynb diff --git a/notebooks/accelerated-worker.ipynb b/notebooks/accelerated-worker.ipynb new file mode 100644 index 0000000..041438f --- /dev/null +++ b/notebooks/accelerated-worker.ipynb @@ -0,0 +1,308 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dask.distributed import Client, Scheduler, Worker" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Close the previous scheduler and all associated workers if present\n", + "try:\n", + " client.shutdown()\n", + " client.close()\n", + "except:\n", + " print('Could not shut down old client, was there any to begin with?')\n", + "\n", + "# Start a client in local cluster mode and expose the underlying scheduler\n", + "client = Client()\n", + "scheduler = client.cluster.scheduler\n", + "print('Dashboard available at', client.dashboard_link)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "no_value = \"--no-value-sentinel--\"\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "# Create an accelerated worker class based on the original worker class\n", + "class AcceleratedWorker(Worker):\n", + " pass\n", + " # def add_task(\n", + " # self,\n", + " # key,\n", + " # function=None,\n", + " # args=None,\n", + " # kwargs=None,\n", + " # task=no_value,\n", + " # who_has=None,\n", + " # nbytes=None,\n", + " # priority=None,\n", + " # duration=None,\n", + " # resource_restrictions=None,\n", + " # actor=False,\n", + " # **kwargs2,\n", + " # ):\n", + " # logger.info('TTTTESTSETSTESTSTETS')\n", + " # super(AcceleratedWorker, self).add_task(\n", + " # key,\n", + " # function,\n", + " # args,\n", + " # kwargs,\n", + " # task,\n", + " # who_has,\n", + " # nbytes,\n", + " # priority,\n", + " # duration,\n", + " # resource_restrictions,\n", + " # actor,\n", + " # **kwargs2,\n", + " # )\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "def install_signal_handlers(loop=None, cleanup=None):\n", + " \"\"\"\n", + " Install global signal handlers to halt the Tornado IOLoop in case of\n", + " a SIGINT or SIGTERM. *cleanup* is an optional callback called,\n", + " before the loop stops, with a single signal number argument.\n", + " \"\"\"\n", + " import signal\n", + "\n", + " loop = loop or IOLoop.current()\n", + "\n", + " old_handlers = {}\n", + "\n", + " def handle_signal(sig, frame):\n", + " async def cleanup_and_stop():\n", + " try:\n", + " if cleanup is not None:\n", + " await cleanup(sig)\n", + " finally:\n", + " loop.stop()\n", + "\n", + " loop.add_callback_from_signal(cleanup_and_stop)\n", + " # Restore old signal handler to allow for a quicker exit\n", + " # if the user sends the signal again.\n", + " signal.signal(sig, old_handlers[sig])\n", + "\n", + " for sig in [signal.SIGINT, signal.SIGTERM]:\n", + " old_handlers[sig] = signal.signal(sig, handle_signal)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Start a new worker based on the AcceleratedWorker class\n", + "# This worker automatically connects to the scheduler and gets added to the worker pool\n", + "# accelerated_worker = await Worker(\n", + "# scheduler.address,\n", + "# validate=True,\n", + "# nthreads=1,\n", + "# memory_limit=4e9,\n", + "# dashboard=True,\n", + "# name='accelerated'\n", + "# )\n", + "\n", + "from tornado.ioloop import IOLoop\n", + "import asyncio\n", + "import sys\n", + "import signal\n", + "\n", + "kwargs = {\n", + " 'preload': (),\n", + " 'memory_limit': '0',\n", + " 'preload_argv': (),\n", + " 'interface': None,\n", + " 'protocol': None,\n", + " 'reconnect': True,\n", + " 'local_directory': None,\n", + " 'death_timeout': None,\n", + " 'lifetime': None,\n", + " 'lifetime_stagger': '0 seconds',\n", + " 'lifetime_restart': False\n", + "}\n", + "\n", + "loop = IOLoop.current()\n", + "\n", + "accelerated_worker = await Worker(\n", + " scheduler.address,\n", + " scheduler_file=None,\n", + " nthreads=1,\n", + " loop=loop,\n", + " resources=None,\n", + " security={},\n", + " contact_address=None,\n", + " host=None,\n", + " port=None,\n", + " dashboard=True,\n", + " dashboard_address=':0',\n", + " name='accelerated',\n", + " **kwargs\n", + ")\n", + "\n", + "nannies = [accelerated_worker]\n", + "\n", + "nanny = True\n", + "\n", + "async def close_all():\n", + " # Unregister all workers from scheduler\n", + " if nanny:\n", + " await asyncio.gather(*[n.close(timeout=2) for n in nannies])\n", + "\n", + "signal_fired = False\n", + "\n", + "def on_signal(signum):\n", + " nonlocal signal_fired\n", + " signal_fired = True\n", + " if signum != signal.SIGINT:\n", + " logger.info(\"Exiting on signal %d\", signum)\n", + " return asyncio.ensure_future(close_all())\n", + "\n", + "async def run():\n", + " await asyncio.gather(*nannies)\n", + " await asyncio.gather(*[n.finished() for n in nannies])\n", + "\n", + "install_signal_handlers(loop, cleanup=on_signal)\n", + "\n", + "try:\n", + " loop.run_sync(run)\n", + "except TimeoutError:\n", + " # We already log the exception in nanny / worker. Don't do it again.\n", + " if not signal_fired:\n", + " logger.info(\"Timed out starting worker\")\n", + " sys.exit(1)\n", + "except KeyboardInterrupt:\n", + " pass\n", + "finally:\n", + " logger.info(\"End worker\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "workers = scheduler.workers\n", + "for worker in workers:\n", + " print(workers[worker].nanny)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Remove the non-accelerated workers\n", + "workers = scheduler.workers\n", + "for worker in workers:\n", + " if worker != accelerated_worker.address:\n", + " await scheduler.remove_worker(address=worker)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Define a simple function and\n", + "# submit a future on the client\n", + "\n", + "# Increment integer values by 1\n", + "def inc(x):\n", + " return x + 1\n", + "\n", + "# Y holds the future to the result\n", + "y = client.submit(inc, 1211)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Print the result of the future when it is ready\n", + "y.result()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 80a9a941ccd817d54c1b27b417f1fe1050697fe2 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Fri, 23 Apr 2021 14:46:21 +0200 Subject: [PATCH 08/43] Ignore all dask-worker-space directories in project --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a8db7a0..8fef6f3 100644 --- a/.gitignore +++ b/.gitignore @@ -153,4 +153,4 @@ cython_debug/ profiler/*.txt # Dask distributed -/dask-worker-space \ No newline at end of file +dask-worker-space/ \ No newline at end of file From fba0e9fbee2261a70894e63db489d826e1c42939 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Fri, 23 Apr 2021 14:46:52 +0200 Subject: [PATCH 09/43] Added tornado and flake8 to required packages --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3e881c8..8f1d139 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,8 @@ 'xeger', 'progressbar2', 'matplotlib', - 'jupyter_contrib_nbextensions' + 'jupyter_contrib_nbextensions', + 'flake8', + 'tornado' ] -) \ No newline at end of file +) From 847a18b41657d33775d1197d8a220fa8de9a587e Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Fri, 23 Apr 2021 14:47:26 +0200 Subject: [PATCH 10/43] Basic setup for an accelerated worker in dask distributed --- dask_accelerated_worker/__init__.py | 0 dask_accelerated_worker/accelerated_worker.py | 41 ++++++ dask_accelerated_worker/main.py | 50 ++++++++ dask_accelerated_worker/start_worker.py | 117 ++++++++++++++++++ 4 files changed, 208 insertions(+) create mode 100644 dask_accelerated_worker/__init__.py create mode 100644 dask_accelerated_worker/accelerated_worker.py create mode 100644 dask_accelerated_worker/main.py create mode 100644 dask_accelerated_worker/start_worker.py diff --git a/dask_accelerated_worker/__init__.py b/dask_accelerated_worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dask_accelerated_worker/accelerated_worker.py b/dask_accelerated_worker/accelerated_worker.py new file mode 100644 index 0000000..2f4f224 --- /dev/null +++ b/dask_accelerated_worker/accelerated_worker.py @@ -0,0 +1,41 @@ +import logging +from dask.distributed import Worker +from dask.distributed import worker + +logger = logging.getLogger(__name__) + + +# Create an accelerated worker class based on the original worker class +class AcceleratedWorker(Worker): + + def add_task( + self, + key, + function=None, + args=None, + kwargs=None, + task=worker.no_value, + who_has=None, + nbytes=None, + priority=None, + duration=None, + resource_restrictions=None, + actor=False, + **kwargs2, + ): + # TODO: substitute fpga accelerated op in task graph + + super().add_task( + key, + function, + args, + kwargs, + task, + who_has, + nbytes, + priority, + duration, + resource_restrictions, + actor, + **kwargs2, + ) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py new file mode 100644 index 0000000..5ec2ce0 --- /dev/null +++ b/dask_accelerated_worker/main.py @@ -0,0 +1,50 @@ +# TODO: include the testing and functionalities of this file into main.py + +from dask.distributed import Client +import asyncio +import logging +logger = logging.getLogger(__name__) + + +def main(): + # Start a client in local cluster mode and expose the underlying scheduler + client = Client() + scheduler = client.cluster.scheduler + print('Dashboard available at', client.dashboard_link) + + loop = asyncio.get_event_loop() + + input("Press Enter to remove non accelerated workers...") + + # Remove the non-accelerated workers + for i in range(3): + workers = scheduler.workers + for worker in workers: + if workers[worker].name != 'accelerated': + loop.run_until_complete( + scheduler.remove_worker(address=worker) + ) + + loop.close() + + input("Press Enter to submit task graph...") + + # Define a simple function and + # submit a future on the client + + # Increment integer values by 1 + def inc(x): + return x + 1 + + # Y holds the future to the result + y = client.submit(inc, 11) + + input("Press Enter to trigger computation...") + print(y.result()) + + input("Press Enter to close the client...") + client.close() + + +if __name__ == '__main__': + main() diff --git a/dask_accelerated_worker/start_worker.py b/dask_accelerated_worker/start_worker.py new file mode 100644 index 0000000..b6dc8e9 --- /dev/null +++ b/dask_accelerated_worker/start_worker.py @@ -0,0 +1,117 @@ +from dask_accelerated_worker.accelerated_worker import AcceleratedWorker +from tornado.ioloop import IOLoop +import asyncio +import sys +import signal +import logging +logger = logging.getLogger(__name__) + +scheduler_address = 'tcp://127.0.0.1:38509' + + +def install_signal_handlers(loop=None, cleanup=None): + """ + Install global signal handlers to halt the Tornado IOLoop in case of + a SIGINT or SIGTERM. *cleanup* is an optional callback called, + before the loop stops, with a single signal number argument. + """ + import signal + + loop = loop or IOLoop.current() + + old_handlers = {} + + def handle_signal(sig, frame): + async def cleanup_and_stop(): + try: + if cleanup is not None: + await cleanup(sig) + finally: + loop.stop() + + loop.add_callback_from_signal(cleanup_and_stop) + # Restore old signal handler to allow for a quicker exit + # if the user sends the signal again. + signal.signal(sig, old_handlers[sig]) + + for sig in [signal.SIGINT, signal.SIGTERM]: + old_handlers[sig] = signal.signal(sig, handle_signal) + + +def main(): + + # Start a new worker based on the AcceleratedWorker class + # This worker automatically connects to the scheduler and gets added to the worker pool + kwargs = { + 'preload': (), + 'memory_limit': '0', + 'preload_argv': (), + 'interface': None, + 'protocol': None, + 'reconnect': True, + 'local_directory': None, + 'death_timeout': None, + 'lifetime': None, + 'lifetime_stagger': '0 seconds', + 'lifetime_restart': False + } + + loop = IOLoop.current() + + async_loop = asyncio.get_event_loop() + accelerated_worker = async_loop.run_until_complete( + AcceleratedWorker( + scheduler_address, + scheduler_file=None, + nthreads=1, + loop=loop, + resources=None, + security={}, + contact_address=None, + host=None, + port=None, + dashboard=True, + dashboard_address=':0', + name='accelerated', + **kwargs + ) + ) + + nannies = [accelerated_worker] + nanny = True + + async def close_all(): + # Unregister all workers from scheduler + if nanny: + await asyncio.gather(*[n.close(timeout=2) for n in nannies]) + + signal_fired = False + + def on_signal(signum): + nonlocal signal_fired + signal_fired = True + if signum != signal.SIGINT: + logger.info("Exiting on signal %d", signum) + return asyncio.ensure_future(close_all()) + + async def run(): + await asyncio.gather(*nannies) + await asyncio.gather(*[n.finished() for n in nannies]) + + install_signal_handlers(loop, cleanup=on_signal) + + try: + loop.run_sync(run) + except TimeoutError: + # We already log the exception in nanny / worker. Don't do it again. + if not signal_fired: + logger.info("Timed out starting worker") + sys.exit(1) + except KeyboardInterrupt: + pass + finally: + logger.info("End worker") + + +if __name__ == '__main__': + main() From a246de1963dd364aebe325f91eb48f106ecac31c Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Fri, 23 Apr 2021 16:14:21 +0200 Subject: [PATCH 11/43] accelerated_worker.py substitutes re2 operator --- dask_accelerated_worker/accelerated_worker.py | 21 +++++++++++++++- dask_accelerated_worker/main.py | 24 ++++++++++++------- dask_accelerated_worker/start_worker.py | 3 ++- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/dask_accelerated_worker/accelerated_worker.py b/dask_accelerated_worker/accelerated_worker.py index 2f4f224..3aa5970 100644 --- a/dask_accelerated_worker/accelerated_worker.py +++ b/dask_accelerated_worker/accelerated_worker.py @@ -1,6 +1,10 @@ import logging +import re +import pickle from dask.distributed import Worker from dask.distributed import worker +from dask.optimization import SubgraphCallable +from dask_accelerated.operators import CustomFilter logger = logging.getLogger(__name__) @@ -23,7 +27,22 @@ def add_task( actor=False, **kwargs2, ): - # TODO: substitute fpga accelerated op in task graph + regex = re.compile('.*str-match.*') + if re.match(regex, key) != None: + # This task matches the operation we want to perform on fpga + func = pickle.loads(function) + + substitute_op = CustomFilter().custom_re2 + + original_dsk = func.dsk + vals = original_dsk[func.outkey] + vals_args = vals[3] + new_vals_args = (vals_args[0], [['_func', substitute_op], vals_args[1][1]]) + new_vals = (vals[0], vals[1], vals[2], new_vals_args) + new_dsk = {func.outkey: new_vals} + + new_func = SubgraphCallable(new_dsk, func.outkey, func.inkeys, "regex_callable") + function = pickle.dumps(new_func) super().add_task( key, diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 5ec2ce0..5d79c7b 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -1,6 +1,7 @@ # TODO: include the testing and functionalities of this file into main.py from dask.distributed import Client +from dask_accelerated import helpers import asyncio import logging logger = logging.getLogger(__name__) @@ -29,18 +30,23 @@ def main(): input("Press Enter to submit task graph...") - # Define a simple function and - # submit a future on the client + in_size = 64e3 + batch_size = 16e3 - # Increment integer values by 1 - def inc(x): - return x + 1 + # The actual batch size in computed by aggregating + # multiple chunks of 1e3 records into the desired batch size + batch_aggregate = int(batch_size / 1e3) + batch_size = 1e3 - # Y holds the future to the result - y = client.submit(inc, 11) + # Make sure the desired dataset exists + helpers.generate_datasets_if_needed([in_size], batch_size) - input("Press Enter to trigger computation...") - print(y.result()) + lazy_result = helpers.get_lazy_result(in_size, batch_size, batch_aggregate) + graph = lazy_result.__dask_graph__() + + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + + print(res) input("Press Enter to close the client...") client.close() diff --git a/dask_accelerated_worker/start_worker.py b/dask_accelerated_worker/start_worker.py index b6dc8e9..2aba2b2 100644 --- a/dask_accelerated_worker/start_worker.py +++ b/dask_accelerated_worker/start_worker.py @@ -1,4 +1,5 @@ from dask_accelerated_worker.accelerated_worker import AcceleratedWorker +from dask.distributed import Worker from tornado.ioloop import IOLoop import asyncio import sys @@ -6,7 +7,7 @@ import logging logger = logging.getLogger(__name__) -scheduler_address = 'tcp://127.0.0.1:38509' +scheduler_address = 'tcp://127.0.0.1:37983' def install_signal_handlers(loop=None, cleanup=None): From 03d1414ebcaf98dc6de6a71ce0316ca8e8c20313 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Fri, 23 Apr 2021 16:22:53 +0200 Subject: [PATCH 12/43] Fixed bug in which accelerated worker did not get any arguments at custom_re2 op --- dask_accelerated_worker/accelerated_worker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dask_accelerated_worker/accelerated_worker.py b/dask_accelerated_worker/accelerated_worker.py index 3aa5970..53f9903 100644 --- a/dask_accelerated_worker/accelerated_worker.py +++ b/dask_accelerated_worker/accelerated_worker.py @@ -34,14 +34,14 @@ def add_task( substitute_op = CustomFilter().custom_re2 - original_dsk = func.dsk - vals = original_dsk[func.outkey] + dsk = func.dsk + vals = dsk[func.outkey] vals_args = vals[3] new_vals_args = (vals_args[0], [['_func', substitute_op], vals_args[1][1]]) new_vals = (vals[0], vals[1], vals[2], new_vals_args) - new_dsk = {func.outkey: new_vals} + dsk[func.outkey] = new_vals - new_func = SubgraphCallable(new_dsk, func.outkey, func.inkeys, "regex_callable") + new_func = SubgraphCallable(dsk, func.outkey, func.inkeys, "regex_callable") function = pickle.dumps(new_func) super().add_task( From 057d7b8f50f99507e9957aeebaa96e93901680d7 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 25 Apr 2021 14:43:35 +0200 Subject: [PATCH 13/43] Added config for Flake8 --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e15a557 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +exclude = .git,*migrations*,native +max-line-length = 119 \ No newline at end of file From aae071af3227f797d7373c4deda7ed3ffff1b9de Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 25 Apr 2021 14:45:21 +0200 Subject: [PATCH 14/43] Added data_generator dir to Flake8 exclude since it is a submodule --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e15a557..6421db6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ [flake8] -exclude = .git,*migrations*,native +exclude = .git,*migrations*,native,data_generator max-line-length = 119 \ No newline at end of file From a68b9994fa554632c8b3c35058c09184608ca1fe Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 25 Apr 2021 14:55:07 +0200 Subject: [PATCH 15/43] Added workflow to check flake8 on PR --- .github/workflows/lint.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..d3d9d9b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,19 @@ +name: lint + +on: [pull_request] + +jobs: + flake8: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: grantmcconnaughey/lintly-flake8-github-action@v1.0 + with: + # The GitHub API token to create reviews with + token: ${{ secrets.GITHUB_TOKEN }} + # Fail if "new" violations detected or "any", default "new" + failIf: new + # Additional arguments to pass to flake8, default "." (current directory) + args: "" \ No newline at end of file From 81e90c3d859573b07d3943da836b4ba8b98e788f Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 25 Apr 2021 15:17:49 +0200 Subject: [PATCH 16/43] Fixed Flake8 warnings --- benchmark/benchmarks.py | 3 -- benchmark/helpers.py | 12 +++++++- benchmark/main.py | 30 +++++++++++++++---- benchmark/pickler.py | 2 +- dask_accelerated/helpers.py | 14 +++++++-- dask_accelerated/operators.py | 3 -- dask_accelerated/optimization.py | 8 +++-- dask_accelerated_worker/accelerated_worker.py | 2 +- dask_accelerated_worker/start_worker.py | 2 +- profiler/main.py | 3 +- test/test_re2.py | 3 +- test/test_tidre.py | 3 +- 12 files changed, 58 insertions(+), 27 deletions(-) diff --git a/benchmark/benchmarks.py b/benchmark/benchmarks.py index 93d534d..6effcba 100644 --- a/benchmark/benchmarks.py +++ b/benchmark/benchmarks.py @@ -124,6 +124,3 @@ def benchmark_tidre_in_size(in_sizes, batch_size, batch_aggregate, repeats): } return data, name - - - diff --git a/benchmark/helpers.py b/benchmark/helpers.py index 671c22c..831a130 100644 --- a/benchmark/helpers.py +++ b/benchmark/helpers.py @@ -2,7 +2,17 @@ from dask_accelerated import helpers -def run_repeats(in_size, batch_size, batch_aggregate, repeats, key, vanilla_filter, re2_filter, tidre_filter=None, tidre_filter_unaligned=None): +def run_repeats( + in_size, + batch_size, + batch_aggregate, + repeats, + key, + vanilla_filter, + re2_filter, + tidre_filter=None, + tidre_filter_unaligned=None +): # Single run to mitigate caching effects (res, dur) = helpers.run_vanilla(in_size, batch_size, batch_aggregate) diff --git a/benchmark/main.py b/benchmark/main.py index a722b6d..fecdb2b 100644 --- a/benchmark/main.py +++ b/benchmark/main.py @@ -13,6 +13,7 @@ args = parser.parse_args() + def benchmark_re2(in_sizes, batch_aggregates, repeats): # Constants when varying a single parameter @@ -28,10 +29,20 @@ def benchmark_re2(in_sizes, batch_aggregates, repeats): data = {} - (benchmark_data, benchmark_name) = bench.benchmark_re2_in_size(in_sizes, constant_batch_size_in_benchmark, constant_batch_aggregate, repeats) + (benchmark_data, benchmark_name) = bench.benchmark_re2_in_size( + in_sizes, + constant_batch_size_in_benchmark, + constant_batch_aggregate, + repeats + ) data[benchmark_name] = benchmark_data - (benchmark_data, benchmark_name) = bench.benchmark_re2_batch_size(constant_in_size, constant_batch_size_batch_benchmark, batch_aggregates, repeats) + (benchmark_data, benchmark_name) = bench.benchmark_re2_batch_size( + constant_in_size, + constant_batch_size_batch_benchmark, + batch_aggregates, + repeats + ) data[benchmark_name] = benchmark_data benchmark_helpers.print_and_store_with_or_without_tidre(data, False) @@ -52,10 +63,20 @@ def benchmark_tidre(in_sizes, batch_aggregates, repeats): data = {} - (benchmark_data, benchmark_name) = bench.benchmark_tidre_in_size(in_sizes, constant_batch_size_in_benchmark, constant_batch_aggregate, repeats) + (benchmark_data, benchmark_name) = bench.benchmark_tidre_in_size( + in_sizes, + constant_batch_size_in_benchmark, + constant_batch_aggregate, + repeats + ) data[benchmark_name] = benchmark_data - (benchmark_data, benchmark_name) = bench.benchmark_tidre_batch_size(constant_in_size, constant_batch_size_batch_benchmark, batch_aggregates, repeats) + (benchmark_data, benchmark_name) = bench.benchmark_tidre_batch_size( + constant_in_size, + constant_batch_size_batch_benchmark, + batch_aggregates, + repeats + ) data[benchmark_name] = benchmark_data benchmark_helpers.print_and_store_with_or_without_tidre(data, True) @@ -75,4 +96,3 @@ def benchmark_tidre(in_sizes, batch_aggregates, repeats): end = time.time() print("Ran all benchmarks in ", (end - start) / 60, " minutes") - diff --git a/benchmark/pickler.py b/benchmark/pickler.py index 5305868..d6fb4d8 100644 --- a/benchmark/pickler.py +++ b/benchmark/pickler.py @@ -21,4 +21,4 @@ def load_from_notebooks(): with open(data_root + 'data.pickle', 'rb') as f: data = pickle.load(f) - return data \ No newline at end of file + return data diff --git a/dask_accelerated/helpers.py b/dask_accelerated/helpers.py index 69ad7e0..9fdc486 100644 --- a/dask_accelerated/helpers.py +++ b/dask_accelerated/helpers.py @@ -110,7 +110,9 @@ def get_lazy_result(in_size, batch_size, split_row_groups): parquet_engine = "pyarrow" # Valid engines: ['fastparquet', 'pyarrow', 'pyarrow-dataset', 'pyarrow-legacy'] file_root = "../data_generator/diving/data-" file_ext = ".parquet" - regex = '.*[tT][eE][rR][aA][tT][iI][dD][eE][ \t\n]+[dD][iI][vV][iI][nN][gG][ \t\n]+([sS][uU][bB])+[sS][uU][rR][fF][aA][cC][eE].*' + regex = '.*[tT][eE][rR][aA][tT][iI][dD][eE][ \t\n]+' \ + '[dD][iI][vV][iI][nN][gG][ \t\n]+' \ + '([sS][uU][bB])+[sS][uU][rR][fF][aA][cC][eE].*' # Load the dataframe columns = ["value", "string"] @@ -151,7 +153,11 @@ def run_and_record_durations(dsk, result, substitute_operator): filter_durations = np.array(substitute_operator.durations) durations = construct_durations(total_duration_in_seconds, filter_durations) - print("Computed ", res, " in ", total_duration_in_seconds, " seconds\tfilter: ", durations['filter']['total'], " seconds") + print( + "Computed ", res, + " in ", total_duration_in_seconds, " seconds", + "\tfilter: ", durations['filter']['total'], " seconds" + ) return res, durations @@ -191,7 +197,9 @@ def generate_datasets_if_needed(sizes, chunksize=1e6): print("Missing datasets found, these will be generated") match_percentage = 0.05 data_length = 100 - regex = '.*[tT][eE][rR][aA][tT][iI][dD][eE][ \t\n]+[dD][iI][vV][iI][nN][gG][ \t\n]+([sS][uU][bB])+[sS][uU][rR][fF][aA][cC][eE].*' + regex = '.*[tT][eE][rR][aA][tT][iI][dD][eE][ \t\n]+' \ + '[dD][iI][vV][iI][nN][gG][ \t\n]+' \ + '([sS][uU][bB])+[sS][uU][rR][fF][aA][cC][eE].*' parquet_chunksize = chunksize parquet_compression = 'none' diff --git a/dask_accelerated/operators.py b/dask_accelerated/operators.py index 5bfb2ed..83f3536 100644 --- a/dask_accelerated/operators.py +++ b/dask_accelerated/operators.py @@ -150,9 +150,6 @@ def custom_tidre_unaligned(self, obj, accessor, attr, args, kwargs): # The number of records in the current batch number_of_records = obj.size - # The regular expression to be matched - regex = args[0] - # Add some padding to the pandas series, which will be removed from the buffers later obj_with_padding = pandas.concat([pandas.Series(["a"]), obj]) arr = pyarrow.Array.from_pandas(obj_with_padding) diff --git a/dask_accelerated/optimization.py b/dask_accelerated/optimization.py index bf4c1be..ee8abbb 100644 --- a/dask_accelerated/optimization.py +++ b/dask_accelerated/optimization.py @@ -1,6 +1,7 @@ import re from dask.optimization import SubgraphCallable + # Unwrap the corresponding subgraph_callable in the task graph in order to insert a custom function def compute_substitute(dsk, key, custom): str_match = dsk[(key, 0)] @@ -16,6 +17,7 @@ def compute_substitute(dsk, key, custom): return SubgraphCallable(new_dsk, call.outkey, call.inkeys, "regex_callable") + # Substitute all string match operators in the graph with the custom re2 operator def optimize_graph_re2(graph, substitute_function): @@ -27,7 +29,7 @@ def optimize_graph_re2(graph, substitute_function): # This key is used to target one of the operators in the task graph # from which the regex_callable will be constructed for key in graph.keys(): - if re.match(regex, key[0]) != None: + if re.match(regex, key[0]) is not None: key = key[0] # The keys are tuples and the operator name is the first value break @@ -39,9 +41,9 @@ def optimize_graph_re2(graph, substitute_function): # Substitute the regex_callable if the operator name matches the str-match pattern for k in dsk: - if re.match(regex, k[0]) != None: + if re.match(regex, k[0]) is not None: target_op = list(dsk[k]) target_op[0] = regex_callable dsk[k] = tuple(target_op) - return dsk \ No newline at end of file + return dsk diff --git a/dask_accelerated_worker/accelerated_worker.py b/dask_accelerated_worker/accelerated_worker.py index 53f9903..a74b7ac 100644 --- a/dask_accelerated_worker/accelerated_worker.py +++ b/dask_accelerated_worker/accelerated_worker.py @@ -28,7 +28,7 @@ def add_task( **kwargs2, ): regex = re.compile('.*str-match.*') - if re.match(regex, key) != None: + if re.match(regex, key) is not None: # This task matches the operation we want to perform on fpga func = pickle.loads(function) diff --git a/dask_accelerated_worker/start_worker.py b/dask_accelerated_worker/start_worker.py index 2aba2b2..36c6b88 100644 --- a/dask_accelerated_worker/start_worker.py +++ b/dask_accelerated_worker/start_worker.py @@ -1,5 +1,5 @@ from dask_accelerated_worker.accelerated_worker import AcceleratedWorker -from dask.distributed import Worker +# from dask.distributed import Worker from tornado.ioloop import IOLoop import asyncio import sys diff --git a/profiler/main.py b/profiler/main.py index 7b8b23a..1cbb062 100644 --- a/profiler/main.py +++ b/profiler/main.py @@ -12,6 +12,7 @@ args = parser.parse_args() + def profile_and_save(func, in_size, batch_size, out_file): # Create a new profiler and enable it, @@ -45,5 +46,3 @@ def profile_and_save(func, in_size, batch_size, out_file): if args.tidre: profile_and_save(helpers.run_tidre, in_size, batch_size, 'tidre_prof.txt') - - diff --git a/test/test_re2.py b/test/test_re2.py index b914ed9..791ef87 100644 --- a/test/test_re2.py +++ b/test/test_re2.py @@ -21,7 +21,6 @@ def test_small_input(self): """ self.run_and_assert_equal(16e3, 32e3) - def test_large_input(self): """ Test that the re2 implementation computes the same @@ -50,7 +49,6 @@ def test_small_input(self): """ self.run_and_assert_equal(16e3, 32e3) - def test_large_input(self): """ Test that the re2 implementation computes the same @@ -59,5 +57,6 @@ def test_large_input(self): """ self.run_and_assert_equal(32e3, 16e3) + if __name__ == '__main__': unittest.main() diff --git a/test/test_tidre.py b/test/test_tidre.py index 431d791..7e34505 100644 --- a/test/test_tidre.py +++ b/test/test_tidre.py @@ -21,7 +21,6 @@ def test_small_input(self): """ self.run_and_assert_equal(16e3, 32e3) - def test_large_input(self): """ Test that the tidre implementation computes the same @@ -50,7 +49,6 @@ def test_small_input(self): """ self.run_and_assert_equal(16e3, 32e3) - def test_large_input(self): """ Test that the tidre implementation computes the same @@ -59,5 +57,6 @@ def test_large_input(self): """ self.run_and_assert_equal(32e3, 16e3) + if __name__ == '__main__': unittest.main() From bc449fe4cba9d3c400fb179dc5b82068b0c1fd24 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 25 Apr 2021 15:27:28 +0200 Subject: [PATCH 17/43] Added linter action badge to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f9ed035..016bef1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Dask Accelerated [![test](https://github.com/teratide/dask-accelerated/actions/workflows/test.yml/badge.svg)](https://github.com/teratide/dask-accelerated/actions/workflows/test.yml) +[![lint](https://github.com/teratide/dask-accelerated/actions/workflows/lint.yml/badge.svg)](https://github.com/teratide/dask-accelerated/actions/workflows/lint.yml) An accelerated version of Dask which substitutes operators in the Dask task graph with an accelerated version. This new operator can do it's evaluation using native libraries or by offloading the computation to an FPGA accelerator. From ca2f51b5f00378ec52b5c303f0f3b8ddba940781 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Mon, 26 Apr 2021 12:13:20 +0200 Subject: [PATCH 18/43] Added simple benchmark for multiple accelerated workers to main.py --- dask_accelerated_worker/main.py | 57 +++++++++++++---- notebooks/plots-worker.ipynb | 109 ++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 notebooks/plots-worker.ipynb diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 5d79c7b..a5bcf3a 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -2,6 +2,9 @@ from dask.distributed import Client from dask_accelerated import helpers +import pickle +from datetime import datetime +import time import asyncio import logging logger = logging.getLogger(__name__) @@ -18,35 +21,61 @@ def main(): input("Press Enter to remove non accelerated workers...") # Remove the non-accelerated workers + accelerated_names = [ + 'accelerated-0', + 'accelerated-1', + 'accelerated-2', + 'accelerated-3' + ] for i in range(3): workers = scheduler.workers for worker in workers: - if workers[worker].name != 'accelerated': + if workers[worker].name not in accelerated_names: loop.run_until_complete( scheduler.remove_worker(address=worker) ) loop.close() - input("Press Enter to submit task graph...") + input("Press Enter to perform benchmarks...") - in_size = 64e3 - batch_size = 16e3 - - # The actual batch size in computed by aggregating - # multiple chunks of 1e3 records into the desired batch size - batch_aggregate = int(batch_size / 1e3) + in_sizes = [1e3, 2e3, 4e3, 8e3, 16e3, 32e3, 64e3, 128e3, 256e3, 512e3, 1024e3, 2048e3, 4096e3] batch_size = 1e3 + batch_aggregate = 16 + repeats = 3 + + data = {} + + for in_size in in_sizes: + + # Make sure the desired dataset exists + helpers.generate_datasets_if_needed([in_size], batch_size) + + lazy_result = helpers.get_lazy_result(in_size, batch_size, batch_aggregate) + graph = lazy_result.__dask_graph__() + + data[in_size] = 0 + + for i in range(repeats): + + start = time.time() + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + end = time.time() - # Make sure the desired dataset exists - helpers.generate_datasets_if_needed([in_size], batch_size) + duration = end - start + data[in_size] += duration + print('Computed ', res, ' in ', duration, ' seconds') - lazy_result = helpers.get_lazy_result(in_size, batch_size, batch_aggregate) - graph = lazy_result.__dask_graph__() + data[in_size] = data[in_size] / repeats - res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + workers = scheduler.workers - print(res) + timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") + # Add timestamp to data + data['timestamp'] = timestamp + data_root = '../notebooks/' + with open(data_root + 'data-' + str(len(workers)) + '.pickle', 'wb') as f: + pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) input("Press Enter to close the client...") client.close() diff --git a/notebooks/plots-worker.ipynb b/notebooks/plots-worker.ipynb new file mode 100644 index 0000000..89a31ae --- /dev/null +++ b/notebooks/plots-worker.ipynb @@ -0,0 +1,109 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from shutil import copyfile\n", + "import pickle" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "data = {}\n", + "\n", + "for i in range(1, 5):\n", + " with open('./data-' + str(i) + '.pickle', 'rb') as f:\n", + " data[i] = pickle.load(f)\n", + "\n", + "print(data[1]['timestamp'])\n", + "out_dir = './plots/workers/' + data[1]['timestamp']\n", + "\n", + "# Create the directory if it does not yet exist\n", + "if not os.path.exists(out_dir):\n", + " os.makedirs(out_dir)\n", + "# Copy the data file to the out dir as a backup\n", + "for i in range(1, 5):\n", + " data[i].pop('timestamp', None)\n", + " copyfile('./data-' + str(i) + '.pickle', out_dir + '/data-' + str(i) + '.pickle')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "x_values = ['1K', '2K', '4K', '8K', '16K', '32K', '64K', '128K', '256K', '512K', '1M', '2M', '4M']\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "plt.plot(x_values, list(data[1].values()), color='b', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_values, list(data[2].values()), color='r', marker='^', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_values, list(data[3].values()), color='y', marker='>', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_values, list(data[4].values()), color='g', marker='x', label='4 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Number of records')\n", + "plt.ylabel('Total query runtime (seconds)')\n", + "\n", + "plt.xticks(x_values)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('RE2 accelerated workers')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/re2_workers_in_sizes.png')\n", + "\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 38b049e8c30b3f478a896cc95f41ac979c8fae9e Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Wed, 28 Apr 2021 11:26:10 +0200 Subject: [PATCH 19/43] Added tabulate to plots.ipynb --- notebooks/plots.ipynb | 86 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/notebooks/plots.ipynb b/notebooks/plots.ipynb index db43d8d..1caaab9 100644 --- a/notebooks/plots.ipynb +++ b/notebooks/plots.ipynb @@ -10,7 +10,8 @@ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from shutil import copyfile\n", - "from benchmark import pickler" + "from benchmark import pickler\n", + "from tabulate import tabulate" ] }, { @@ -42,7 +43,7 @@ "print(\"Run using tidre: \", tidre)\n", "print(\"Data generated on\", timestamp)\n", "\n", - "# Determing output path for generated plots\n", + "# Determine output path for generated plots\n", "out_dir = './plots/'\n", "if tidre:\n", " out_dir += 'tidre/'\n", @@ -58,6 +59,87 @@ "copyfile('./data.pickle', out_dir + '/data.pickle')" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# All measurements are averaged over 10 runs\n", + "# Before each measurement, a single run is performed and discarded\n", + "# All records contain a string of 100 singly byte ascii characters\n", + "in_sizes = [1e3, 2e3, 4e3, 8e3, 16e3, 32e3, 64e3, 128e3, 256e3, 512e3, 1024e3, 2048e3, 4096e3]\n", + "in_bytes = np.array([x * 100 * 1 for x in in_sizes])\n", + "\n", + "# Varies the number of input records\n", + "# Batch size is kept constant at 1M records\n", + "in_size_runtime = {\n", + " 'in sizes (records)': list(data['in_size']['vanilla_filter'].keys()),\n", + " 'vanilla filter (s)': list(data['in_size']['vanilla_filter'].values()),\n", + " 're2 filter (s)': list(data['in_size']['re2_filter'].values()),\n", + " 'tidre filter (s)': list(data['in_size']['tidre_filter'].values()),\n", + " 'tidre filter unaligned (s)': list(data['in_size']['tidre_filter_unaligned'].values())\n", + "}\n", + "\n", + "in_size_throughput = {\n", + " 'in sizes (records)': list(data['in_size']['vanilla_filter'].keys()),\n", + " 'vanilla filter (bytes/s)': np.divide(in_bytes, list(data['in_size']['vanilla_filter'].values())),\n", + " 're2 filter (bytes/s)': np.divide(in_bytes, list(data['in_size']['re2_filter'].values())),\n", + " 'tidre filter (bytes/s)': np.divide(in_bytes, list(data['in_size']['tidre_filter'].values())),\n", + " 'tidre filter unaligned (bytes/s)': np.divide(in_bytes, list(data['in_size']['tidre_filter_unaligned'].values()))\n", + "}\n", + "\n", + "# Varies the number of records in a single record batch\n", + "# In size is kept constant at 4M records\n", + "batch_size_runtime = {\n", + " 'batch sizes (records)': list(data['batch_size']['vanilla_filter'].keys()),\n", + " 'vanilla filter (s)': list(data['batch_size']['vanilla_filter'].values()),\n", + " 're2 filter (s)': list(data['batch_size']['re2_filter'].values()),\n", + " 'tidre filter (s)': list(data['batch_size']['tidre_filter'].values()),\n", + " 'tidre filter unaligned (s)': list(data['batch_size']['tidre_filter_unaligned'].values())\n", + "}\n", + "\n", + "in_bytes = 4096e3 * 100 * 1\n", + "batch_size_throughput = {\n", + " 'batch sizes (records)': list(data['batch_size']['vanilla_filter'].keys()),\n", + " 'vanilla filter (bytes/s)': np.divide(in_bytes, list(data['batch_size']['vanilla_filter'].values())),\n", + " 're2 filter (bytes/s)': np.divide(in_bytes, list(data['batch_size']['re2_filter'].values())),\n", + " 'tidre filter (bytes/s)': np.divide(in_bytes, list(data['batch_size']['tidre_filter'].values())),\n", + " 'tidre filter unaligned (bytes/s)': np.divide(in_bytes, list(data['batch_size']['tidre_filter_unaligned'].values()))\n", + "}\n", + "\n", + "print('Dask in size benchmark runtime:')\n", + "print(tabulate(in_size_runtime, headers='keys'))\n", + "\n", + "print('\\nDask in size benchmark throughput:')\n", + "print(tabulate(in_size_throughput, headers='keys'))\n", + "\n", + "print('\\n\\nDask batch size runtime:')\n", + "print(tabulate(batch_size_runtime, headers='keys'))\n", + "\n", + "print('\\nDask batch size throughput:')\n", + "print(tabulate(batch_size_throughput, headers='keys'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "speedup_tidre_re2 = data['in_size']['vanilla_filter']['4M'] / data['in_size']['tidre_filter']['4M']\n", + "\n", + "print(speedup_tidre_re2)" + ] + }, { "cell_type": "code", "execution_count": null, From f76ab60038a1988c38cd460307ee6ab1144a8780 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Fri, 30 Apr 2021 14:01:07 +0200 Subject: [PATCH 20/43] Changed CMakeLists.txt ABI to 1 --- native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 4f5db9c..b5d54b4 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -4,7 +4,7 @@ project(dask_native) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0) +set(CMAKE_CXX_FLAGS -D_GLIBCXX_USE_CXX11_ABI=1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) include(FetchContent) From bf727766aca791b545d173fcddc55480008bee50 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 2 May 2021 16:08:51 +0200 Subject: [PATCH 21/43] Streamlined testing on AWS --- dask_accelerated_worker/accelerated_worker.py | 2 +- dask_accelerated_worker/main.py | 60 +++++++++---------- dask_accelerated_worker/start_worker.py | 16 ++++- 3 files changed, 45 insertions(+), 33 deletions(-) diff --git a/dask_accelerated_worker/accelerated_worker.py b/dask_accelerated_worker/accelerated_worker.py index a74b7ac..ead9018 100644 --- a/dask_accelerated_worker/accelerated_worker.py +++ b/dask_accelerated_worker/accelerated_worker.py @@ -32,7 +32,7 @@ def add_task( # This task matches the operation we want to perform on fpga func = pickle.loads(function) - substitute_op = CustomFilter().custom_re2 + substitute_op = CustomFilter().custom_tidre dsk = func.dsk vals = dsk[func.outkey] diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index a5bcf3a..56b7317 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -15,22 +15,17 @@ def main(): client = Client() scheduler = client.cluster.scheduler print('Dashboard available at', client.dashboard_link) + print('Scheduler address: ', scheduler.address) loop = asyncio.get_event_loop() input("Press Enter to remove non accelerated workers...") # Remove the non-accelerated workers - accelerated_names = [ - 'accelerated-0', - 'accelerated-1', - 'accelerated-2', - 'accelerated-3' - ] for i in range(3): workers = scheduler.workers for worker in workers: - if workers[worker].name not in accelerated_names: + if str(workers[worker].name).split('-')[0] != 'accelerated': loop.run_until_complete( scheduler.remove_worker(address=worker) ) @@ -44,40 +39,45 @@ def main(): batch_aggregate = 16 repeats = 3 - data = {} + # Make sure the desired dataset exists + helpers.generate_datasets_if_needed(in_sizes, batch_size) - for in_size in in_sizes: + # Keep running the benchmark until the user quits the client + while True: + data = {} - # Make sure the desired dataset exists - helpers.generate_datasets_if_needed([in_size], batch_size) + for in_size in in_sizes: - lazy_result = helpers.get_lazy_result(in_size, batch_size, batch_aggregate) - graph = lazy_result.__dask_graph__() + lazy_result = helpers.get_lazy_result(in_size, batch_size, batch_aggregate) + graph = lazy_result.__dask_graph__() - data[in_size] = 0 + data[in_size] = 0 - for i in range(repeats): + for i in range(repeats): - start = time.time() - res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) - end = time.time() + start = time.time() + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + end = time.time() - duration = end - start - data[in_size] += duration - print('Computed ', res, ' in ', duration, ' seconds') + duration = end - start + data[in_size] += duration + print('Computed ', res, ' in ', duration, ' seconds') - data[in_size] = data[in_size] / repeats + data[in_size] = data[in_size] / repeats - workers = scheduler.workers + workers = scheduler.workers + + timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") + # Add timestamp to data + data['timestamp'] = timestamp + data_root = '../notebooks/' + with open(data_root + 'data-' + str(len(workers)) + '.pickle', 'wb') as f: + pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) - timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") - # Add timestamp to data - data['timestamp'] = timestamp - data_root = '../notebooks/' - with open(data_root + 'data-' + str(len(workers)) + '.pickle', 'wb') as f: - pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + user_input = input("Press Enter to run again or send 'q' to close the client...") + if user_input == 'q': + break - input("Press Enter to close the client...") client.close() diff --git a/dask_accelerated_worker/start_worker.py b/dask_accelerated_worker/start_worker.py index 36c6b88..8a95c53 100644 --- a/dask_accelerated_worker/start_worker.py +++ b/dask_accelerated_worker/start_worker.py @@ -5,9 +5,17 @@ import sys import signal import logging +import argparse +import time logger = logging.getLogger(__name__) -scheduler_address = 'tcp://127.0.0.1:37983' +# scheduler_address = 'tcp://127.0.0.1:37983' + +parser = argparse.ArgumentParser(description='Dask Accelerated Worker.') +parser.add_argument('scheduler_address', metavar='S', type=str, + help='string containing the ip and port of the scheduler. Example: tcp://127.0.0.1:37983') + +args = parser.parse_args() def install_signal_handlers(loop=None, cleanup=None): @@ -41,6 +49,8 @@ async def cleanup_and_stop(): def main(): + scheduler_address = args.scheduler_address + # Start a new worker based on the AcceleratedWorker class # This worker automatically connects to the scheduler and gets added to the worker pool kwargs = { @@ -57,6 +67,8 @@ def main(): 'lifetime_restart': False } + worker_name = 'accelerated-' + str(time.time()) + loop = IOLoop.current() async_loop = asyncio.get_event_loop() @@ -73,7 +85,7 @@ def main(): port=None, dashboard=True, dashboard_address=':0', - name='accelerated', + name=worker_name, **kwargs ) ) From ba3bbca49b1971582c6b0d11fc9ae2ce1176f206 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 2 May 2021 17:07:30 +0200 Subject: [PATCH 22/43] Added bokeh to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8f1d139..c5192f7 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ 'dask', 'dask[distributed]', 'dask[dataframe]', + 'bokeh', 'jupyter-server-proxy', 'pyarrow', 'pandas', From e94e7b31263c01c0f8c333b5c3fa92537e13aca7 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Wed, 5 May 2021 17:49:58 +0200 Subject: [PATCH 23/43] Added option to spin up vanilla workers from start_worker.py --- dask_accelerated_worker/start_worker.py | 21 +++++--- notebooks/plots-worker.ipynb | 70 +++++++++++++++++++++---- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/dask_accelerated_worker/start_worker.py b/dask_accelerated_worker/start_worker.py index 8a95c53..097b4bf 100644 --- a/dask_accelerated_worker/start_worker.py +++ b/dask_accelerated_worker/start_worker.py @@ -1,5 +1,5 @@ from dask_accelerated_worker.accelerated_worker import AcceleratedWorker -# from dask.distributed import Worker +from dask.distributed import Worker from tornado.ioloop import IOLoop import asyncio import sys @@ -9,11 +9,13 @@ import time logger = logging.getLogger(__name__) -# scheduler_address = 'tcp://127.0.0.1:37983' parser = argparse.ArgumentParser(description='Dask Accelerated Worker.') parser.add_argument('scheduler_address', metavar='S', type=str, help='string containing the ip and port of the scheduler. Example: tcp://127.0.0.1:37983') +parser.add_argument('--accelerated', dest='accelerated', action='store_const', + const=True, default=False, + help='use the accelerated worker implementation. (default: do not use)') args = parser.parse_args() @@ -51,6 +53,13 @@ def main(): scheduler_address = args.scheduler_address + if args.accelerated: + t = AcceleratedWorker + worker_name = 'accelerated-' + str(time.time()) + else: + t = Worker + worker_name = 'vanilla-' + str(time.time()) + # Start a new worker based on the AcceleratedWorker class # This worker automatically connects to the scheduler and gets added to the worker pool kwargs = { @@ -67,13 +76,11 @@ def main(): 'lifetime_restart': False } - worker_name = 'accelerated-' + str(time.time()) - loop = IOLoop.current() async_loop = asyncio.get_event_loop() - accelerated_worker = async_loop.run_until_complete( - AcceleratedWorker( + worker = async_loop.run_until_complete( + t( scheduler_address, scheduler_file=None, nthreads=1, @@ -90,7 +97,7 @@ def main(): ) ) - nannies = [accelerated_worker] + nannies = [worker] nanny = True async def close_all(): diff --git a/notebooks/plots-worker.ipynb b/notebooks/plots-worker.ipynb index 89a31ae..ccb22c7 100644 --- a/notebooks/plots-worker.ipynb +++ b/notebooks/plots-worker.ipynb @@ -25,10 +25,14 @@ "source": [ "data = {}\n", "\n", - "for i in range(1, 5):\n", + "for i in range(1, 2):\n", " with open('./data-' + str(i) + '.pickle', 'rb') as f:\n", " data[i] = pickle.load(f)\n", "\n", + "with open('./data-1-vanilla.pickle', 'rb') as f:\n", + " data_vanilla = pickle.load(f)\n", + "data_vanilla.pop('timestamp', None)\n", + "\n", "print(data[1]['timestamp'])\n", "out_dir = './plots/workers/' + data[1]['timestamp']\n", "\n", @@ -36,9 +40,55 @@ "if not os.path.exists(out_dir):\n", " os.makedirs(out_dir)\n", "# Copy the data file to the out dir as a backup\n", - "for i in range(1, 5):\n", + "for i in range(1, 2):\n", " data[i].pop('timestamp', None)\n", - " copyfile('./data-' + str(i) + '.pickle', out_dir + '/data-' + str(i) + '.pickle')" + " copyfile('./data-' + str(i) + '.pickle', out_dir + '/data-' + str(i) + '.pickle')\n", + "\n", + "copyfile('./data-1-vanilla.pickle', out_dir + '/data-1-vanilla.pickle')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "x_values = ['1K', '2K', '4K', '8K', '16K', '32K', '64K', '128K', '256K', '512K', '1M', '2M', '4M']\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "plt.plot(x_values, list(data_vanilla.values()), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_values, list(data[1].values()), color='b', marker='o', label='accelerated worker', zorder=3)\n", + "# plt.plot(x_values, list(data[2].values()), color='r', marker='^', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_values, list(data[3].values()), color='y', marker='>', label='3 accelerated workers', zorder=3)\n", + "# plt.plot(x_values, list(data[4].values()), color='g', marker='x', label='4 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Number of records')\n", + "plt.ylabel('Total query runtime (seconds)')\n", + "\n", + "plt.xticks(x_values)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('Runtime accelerated worker')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/tidre_workers_in_sizes.png')\n", + "\n", + "plt.show()" ] }, { @@ -56,10 +106,12 @@ "fig = plt.figure(figsize=(9,7))\n", "fig.patch.set_facecolor('white')\n", "\n", - "plt.plot(x_values, list(data[1].values()), color='b', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_values, list(data[2].values()), color='r', marker='^', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_values, list(data[3].values()), color='y', marker='>', label='3 accelerated workers', zorder=3)\n", - "plt.plot(x_values, list(data[4].values()), color='g', marker='x', label='4 accelerated workers', zorder=3)\n", + "speedup = np.divide(list(data_vanilla.values()), list(data[1].values()))\n", + "\n", + "plt.plot(x_values, speedup, color='b', marker='o', label='accelerated worker', zorder=3)\n", + "# plt.plot(x_values, list(data[2].values()), color='r', marker='^', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_values, list(data[3].values()), color='y', marker='>', label='3 accelerated workers', zorder=3)\n", + "# plt.plot(x_values, list(data[4].values()), color='g', marker='x', label='4 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Number of records')\n", @@ -68,7 +120,7 @@ "plt.xticks(x_values)\n", "plt.xticks(rotation=-90)\n", "\n", - "plt.title('RE2 accelerated workers')\n", + "plt.title('Speedup accelerated worker')\n", "\n", "axes = plt.gca()\n", "axes.grid(which='both', axis='y', linestyle='--')\n", @@ -79,7 +131,7 @@ "plt.legend()\n", "\n", "# Save fig as pdf\n", - "plt.savefig(out_dir + '/re2_workers_in_sizes.png')\n", + "plt.savefig(out_dir + '/speedup_tidre_workers_in_sizes.png')\n", "\n", "plt.show()" ] From 92788800c92c59ffa87d5afd0601b856d2e1c092 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Wed, 5 May 2021 17:50:21 +0200 Subject: [PATCH 24/43] Added batch size benchmark --- dask_accelerated_worker/main.py | 64 ++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 56b7317..9ed9f00 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -34,10 +34,12 @@ def main(): input("Press Enter to perform benchmarks...") - in_sizes = [1e3, 2e3, 4e3, 8e3, 16e3, 32e3, 64e3, 128e3, 256e3, 512e3, 1024e3, 2048e3, 4096e3] + const_in_size = 4096e3 + in_sizes = [512e3, 1024e3, 2048e3, 4096e3, 8192e3] batch_size = 1e3 - batch_aggregate = 16 - repeats = 3 + const_batch_aggregate = 1e3 + batch_aggregates = [64, 128, 256, 512, 1024, 2048, 4096, 8192] + repeats = 5 # Make sure the desired dataset exists helpers.generate_datasets_if_needed(in_sizes, batch_size) @@ -45,13 +47,19 @@ def main(): # Keep running the benchmark until the user quits the client while True: data = {} + data_in_size = {} + data_batch_size = {} + # In size benchmark for in_size in in_sizes: - lazy_result = helpers.get_lazy_result(in_size, batch_size, batch_aggregate) + lazy_result = helpers.get_lazy_result(in_size, batch_size, const_batch_aggregate) graph = lazy_result.__dask_graph__() - data[in_size] = 0 + # Dry run + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + + data_in_size[in_size] = 0 for i in range(repeats): @@ -60,24 +68,54 @@ def main(): end = time.time() duration = end - start - data[in_size] += duration - print('Computed ', res, ' in ', duration, ' seconds') + data_in_size[in_size] += duration + print('In: ', in_size, '\tBatch: ', batch_size * const_batch_aggregate, '\tComputed ', res, ' in ', duration, ' seconds') - data[in_size] = data[in_size] / repeats + data_in_size[in_size] = data_in_size[in_size] / repeats - workers = scheduler.workers + # Batch size benchmark + for batch_aggregate in batch_aggregates: + + lazy_result = helpers.get_lazy_result(const_in_size, batch_size, batch_aggregate) + graph = lazy_result.__dask_graph__() + + # Dry run + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + + data_batch_size[batch_aggregate] = 0 + + for i in range(repeats): + start = time.time() + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + end = time.time() + + duration = end - start + data_batch_size[batch_aggregate] += duration + print('In: ', const_in_size, '\tBatch: ', batch_size * batch_aggregate, 'Computed ', res, ' in ', duration, ' seconds') + + data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / repeats + + # Count the number of accelerated workers + accelerated_workers = 0 + for worker in scheduler.workers: + if str(workers[worker].name).split('-')[0] == 'accelerated': + accelerated_workers += 1 + + data[str(accelerated_workers)]['in_size'] = data_in_size + data[str(accelerated_workers)]['batch_size'] = data_batch_size - timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") # Add timestamp to data + timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") data['timestamp'] = timestamp - data_root = '../notebooks/' - with open(data_root + 'data-' + str(len(workers)) + '.pickle', 'wb') as f: - pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) user_input = input("Press Enter to run again or send 'q' to close the client...") if user_input == 'q': break + data_root = '../notebooks/' + with open(data_root + 'data-workers.pickle', 'wb') as f: + pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + client.close() From e69d0f7c2348b174e528922dde19f84c565c38c1 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Wed, 5 May 2021 18:32:44 +0200 Subject: [PATCH 25/43] Specify scheduler address on cli for main.py --- dask_accelerated_worker/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 9ed9f00..437b37f 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -7,12 +7,20 @@ import time import asyncio import logging +import argparse logger = logging.getLogger(__name__) +parser = argparse.ArgumentParser(description='Dask Accelerated Worker.') +parser.add_argument('scheduler_address', metavar='S', type=str, + help='string containing the ip and port of the scheduler. Example: 127.0.0.1:37983') + +args = parser.parse_args() + + def main(): # Start a client in local cluster mode and expose the underlying scheduler - client = Client() + client = Client(args.scheduler_address) scheduler = client.cluster.scheduler print('Dashboard available at', client.dashboard_link) print('Scheduler address: ', scheduler.address) From 08c0880f6e5cfa2e946e64d96e261966f6954068 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 14:37:10 +0200 Subject: [PATCH 26/43] Fixed lintly E503 --- dask_accelerated_worker/main.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 437b37f..9633c49 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -77,7 +77,11 @@ def main(): duration = end - start data_in_size[in_size] += duration - print('In: ', in_size, '\tBatch: ', batch_size * const_batch_aggregate, '\tComputed ', res, ' in ', duration, ' seconds') + print( + 'In: ', in_size, + '\tBatch: ', batch_size * const_batch_aggregate, + '\tComputed ', res, ' in ', duration, ' seconds' + ) data_in_size[in_size] = data_in_size[in_size] / repeats @@ -99,7 +103,11 @@ def main(): duration = end - start data_batch_size[batch_aggregate] += duration - print('In: ', const_in_size, '\tBatch: ', batch_size * batch_aggregate, 'Computed ', res, ' in ', duration, ' seconds') + print( + 'In: ', const_in_size, + '\tBatch: ', batch_size * batch_aggregate, + 'Computed ', res, ' in ', duration, ' seconds' + ) data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / repeats From f70761ea18728fbc7d640c141de88c29a1eb5ecb Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 16:34:40 +0200 Subject: [PATCH 27/43] Start scheduler explicitly such that cluster is not running in local mode --- dask_accelerated_worker/main.py | 86 +++++++++++++++++++++---- dask_accelerated_worker/start_worker.py | 30 +-------- dask_accelerated_worker/utils.py | 30 +++++++++ 3 files changed, 105 insertions(+), 41 deletions(-) create mode 100644 dask_accelerated_worker/utils.py diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 9633c49..fed2839 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -1,31 +1,93 @@ # TODO: include the testing and functionalities of this file into main.py -from dask.distributed import Client +from dask.distributed import Client, Scheduler from dask_accelerated import helpers +from dask_accelerated_worker.utils import install_signal_handlers import pickle from datetime import datetime import time import asyncio import logging -import argparse +from tornado.ioloop import IOLoop +from threading import Thread logger = logging.getLogger(__name__) -parser = argparse.ArgumentParser(description='Dask Accelerated Worker.') -parser.add_argument('scheduler_address', metavar='S', type=str, - help='string containing the ip and port of the scheduler. Example: 127.0.0.1:37983') - -args = parser.parse_args() +def get_scheduler(): + kwargs = { + 'preload': (), + 'preload_argv': (), + 'interface': None, + 'protocol': None, + 'scheduler_file': '', + 'idle_timeout': None + } + + loop = IOLoop.current() + sec = {} + host = '' + port = 8786 + dashboard = True + dashboard_address = 8787 + dashboard_prefix = '' + + scheduler = Scheduler( + loop=loop, + security=sec, + host=host, + port=port, + dashboard=dashboard, + dashboard_address=dashboard_address, + http_prefix=dashboard_prefix, + **kwargs + ) + + return scheduler, loop + + +def run_scheduler(scheduler, loop): + + async def run(): + await scheduler + await scheduler.finished() + + loop.run_sync(run) + + # install_signal_handlers(loop) + # + # async def run(): + # await scheduler + # await scheduler.finished() + # + # try: + # loop.run_sync(run) + # finally: + # scheduler.stop() + # + # logger.info("End scheduler at %r", scheduler.address) def main(): # Start a client in local cluster mode and expose the underlying scheduler - client = Client(args.scheduler_address) - scheduler = client.cluster.scheduler - print('Dashboard available at', client.dashboard_link) + # client = Client(args.scheduler_address) + # scheduler = client.cluster.scheduler + # print('Dashboard available at', client.dashboard_link) + # print('Scheduler address: ', scheduler.address) + + (scheduler, scheduler_loop) = get_scheduler() + + thread = Thread(target=run_scheduler, args=(scheduler, scheduler_loop, )) + thread.start() + + time.sleep(5) + + # print('Dashboard available at', scheduler.dashboard_link) print('Scheduler address: ', scheduler.address) - loop = asyncio.get_event_loop() + client = Client(scheduler.address) + + # loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() input("Press Enter to remove non accelerated workers...") @@ -38,7 +100,7 @@ def main(): scheduler.remove_worker(address=worker) ) - loop.close() + # loop.close() input("Press Enter to perform benchmarks...") diff --git a/dask_accelerated_worker/start_worker.py b/dask_accelerated_worker/start_worker.py index 097b4bf..bfe4285 100644 --- a/dask_accelerated_worker/start_worker.py +++ b/dask_accelerated_worker/start_worker.py @@ -1,6 +1,7 @@ from dask_accelerated_worker.accelerated_worker import AcceleratedWorker from dask.distributed import Worker from tornado.ioloop import IOLoop +from dask_accelerated_worker.utils import install_signal_handlers import asyncio import sys import signal @@ -20,35 +21,6 @@ args = parser.parse_args() -def install_signal_handlers(loop=None, cleanup=None): - """ - Install global signal handlers to halt the Tornado IOLoop in case of - a SIGINT or SIGTERM. *cleanup* is an optional callback called, - before the loop stops, with a single signal number argument. - """ - import signal - - loop = loop or IOLoop.current() - - old_handlers = {} - - def handle_signal(sig, frame): - async def cleanup_and_stop(): - try: - if cleanup is not None: - await cleanup(sig) - finally: - loop.stop() - - loop.add_callback_from_signal(cleanup_and_stop) - # Restore old signal handler to allow for a quicker exit - # if the user sends the signal again. - signal.signal(sig, old_handlers[sig]) - - for sig in [signal.SIGINT, signal.SIGTERM]: - old_handlers[sig] = signal.signal(sig, handle_signal) - - def main(): scheduler_address = args.scheduler_address diff --git a/dask_accelerated_worker/utils.py b/dask_accelerated_worker/utils.py new file mode 100644 index 0000000..81442f7 --- /dev/null +++ b/dask_accelerated_worker/utils.py @@ -0,0 +1,30 @@ +from tornado.ioloop import IOLoop + + +def install_signal_handlers(loop=None, cleanup=None): + """ + Install global signal handlers to halt the Tornado IOLoop in case of + a SIGINT or SIGTERM. *cleanup* is an optional callback called, + before the loop stops, with a single signal number argument. + """ + import signal + + loop = loop or IOLoop.current() + + old_handlers = {} + + def handle_signal(sig, frame): + async def cleanup_and_stop(): + try: + if cleanup is not None: + await cleanup(sig) + finally: + loop.stop() + + loop.add_callback_from_signal(cleanup_and_stop) + # Restore old signal handler to allow for a quicker exit + # if the user sends the signal again. + signal.signal(sig, old_handlers[sig]) + + for sig in [signal.SIGINT, signal.SIGTERM]: + old_handlers[sig] = signal.signal(sig, handle_signal) \ No newline at end of file From 785c9179f873ae3bffe72785345ff80ead4f8b91 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 16:49:45 +0200 Subject: [PATCH 28/43] Cleaned main.py and extracted scheduler specific methods to helpers.py --- dask_accelerated_worker/helpers.py | 65 ++++++++++++++++ dask_accelerated_worker/main.py | 115 ++++++----------------------- 2 files changed, 87 insertions(+), 93 deletions(-) create mode 100644 dask_accelerated_worker/helpers.py diff --git a/dask_accelerated_worker/helpers.py b/dask_accelerated_worker/helpers.py new file mode 100644 index 0000000..2b1bfb3 --- /dev/null +++ b/dask_accelerated_worker/helpers.py @@ -0,0 +1,65 @@ +from dask.distributed import Scheduler +from tornado.ioloop import IOLoop +import asyncio + + +def get_scheduler(): + kwargs = { + 'preload': (), + 'preload_argv': (), + 'interface': None, + 'protocol': None, + 'scheduler_file': '', + 'idle_timeout': None + } + + loop = IOLoop.current() + sec = {} + host = '' + port = 8786 + dashboard = True + dashboard_address = 8787 + dashboard_prefix = '' + + scheduler = Scheduler( + loop=loop, + security=sec, + host=host, + port=port, + dashboard=dashboard, + dashboard_address=dashboard_address, + http_prefix=dashboard_prefix, + **kwargs + ) + + return scheduler, loop + + +def run_scheduler(scheduler, loop): + + async def run(): + await scheduler + await scheduler.finished() + + loop.run_sync(run) + + +def remove_non_accelerated_workers(scheduler): + + # New event loop to await async remove worker method + loop = asyncio.new_event_loop() + + # TODO: fix this + # Somehow this does not always work on the first try + # A quick but messy fix is to run it more than once to + # ensure all non-accelerated workers get removed + for i in range(3): + workers = scheduler.workers + for worker in workers: + # All accelerated workers are called 'accelerated-[timestamp]' + if str(workers[worker].name).split('-')[0] != 'accelerated': + loop.run_until_complete( + scheduler.remove_worker(address=worker) + ) + + loop.close() diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index fed2839..b655dcc 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -1,116 +1,45 @@ -# TODO: include the testing and functionalities of this file into main.py - -from dask.distributed import Client, Scheduler +from dask.distributed import Client from dask_accelerated import helpers -from dask_accelerated_worker.utils import install_signal_handlers +from dask_accelerated_worker import helpers as worker_helpers import pickle from datetime import datetime import time -import asyncio -import logging -from tornado.ioloop import IOLoop from threading import Thread +import logging + logger = logging.getLogger(__name__) -def get_scheduler(): - kwargs = { - 'preload': (), - 'preload_argv': (), - 'interface': None, - 'protocol': None, - 'scheduler_file': '', - 'idle_timeout': None - } - - loop = IOLoop.current() - sec = {} - host = '' - port = 8786 - dashboard = True - dashboard_address = 8787 - dashboard_prefix = '' - - scheduler = Scheduler( - loop=loop, - security=sec, - host=host, - port=port, - dashboard=dashboard, - dashboard_address=dashboard_address, - http_prefix=dashboard_prefix, - **kwargs - ) - - return scheduler, loop - - -def run_scheduler(scheduler, loop): - - async def run(): - await scheduler - await scheduler.finished() - - loop.run_sync(run) - - # install_signal_handlers(loop) - # - # async def run(): - # await scheduler - # await scheduler.finished() - # - # try: - # loop.run_sync(run) - # finally: - # scheduler.stop() - # - # logger.info("End scheduler at %r", scheduler.address) +# Benchmark configuration +const_in_size = 4096e3 +in_sizes = [512e3, 1024e3, 2048e3, 4096e3, 8192e3] +batch_size = 1e3 +const_batch_aggregate = 1e3 +batch_aggregates = [64, 128, 256, 512, 1024, 2048, 4096, 8192] +repeats = 5 def main(): - # Start a client in local cluster mode and expose the underlying scheduler - # client = Client(args.scheduler_address) - # scheduler = client.cluster.scheduler - # print('Dashboard available at', client.dashboard_link) - # print('Scheduler address: ', scheduler.address) - - (scheduler, scheduler_loop) = get_scheduler() - thread = Thread(target=run_scheduler, args=(scheduler, scheduler_loop, )) + # Start a dask scheduler + (scheduler, scheduler_loop) = worker_helpers.get_scheduler() + thread = Thread(target=worker_helpers.run_scheduler, args=(scheduler, scheduler_loop,)) thread.start() - time.sleep(5) - - # print('Dashboard available at', scheduler.dashboard_link) - print('Scheduler address: ', scheduler.address) + # Wait 1 second to ensure the scheduler is running + time.sleep(1) + # Start a client and connect it to the scheduler client = Client(scheduler.address) - # loop = asyncio.get_event_loop() - loop = asyncio.new_event_loop() - - input("Press Enter to remove non accelerated workers...") - - # Remove the non-accelerated workers - for i in range(3): - workers = scheduler.workers - for worker in workers: - if str(workers[worker].name).split('-')[0] != 'accelerated': - loop.run_until_complete( - scheduler.remove_worker(address=worker) - ) + print('Dashboard available at', client.dashboard_link) + print('Scheduler address: ', scheduler.address) - # loop.close() + # input("Press Enter to remove non accelerated workers...") + # worker_helpers.remove_non_accelerated_workers(scheduler) input("Press Enter to perform benchmarks...") - const_in_size = 4096e3 - in_sizes = [512e3, 1024e3, 2048e3, 4096e3, 8192e3] - batch_size = 1e3 - const_batch_aggregate = 1e3 - batch_aggregates = [64, 128, 256, 512, 1024, 2048, 4096, 8192] - repeats = 5 - # Make sure the desired dataset exists helpers.generate_datasets_if_needed(in_sizes, batch_size) @@ -176,7 +105,7 @@ def main(): # Count the number of accelerated workers accelerated_workers = 0 for worker in scheduler.workers: - if str(workers[worker].name).split('-')[0] == 'accelerated': + if str(scheduler.workers[worker].name).split('-')[0] == 'accelerated': accelerated_workers += 1 data[str(accelerated_workers)]['in_size'] = data_in_size From af00024e8f76a9197df7324c48f4ffb374dc147b Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 17:23:09 +0200 Subject: [PATCH 29/43] Remove benchmark dry run --- dask_accelerated_worker/main.py | 4 ++-- notebooks/plots-worker.ipynb | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index b655dcc..ee0ae96 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -56,7 +56,7 @@ def main(): graph = lazy_result.__dask_graph__() # Dry run - res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + # res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) data_in_size[in_size] = 0 @@ -83,7 +83,7 @@ def main(): graph = lazy_result.__dask_graph__() # Dry run - res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + # res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) data_batch_size[batch_aggregate] = 0 diff --git a/notebooks/plots-worker.ipynb b/notebooks/plots-worker.ipynb index ccb22c7..aedcd65 100644 --- a/notebooks/plots-worker.ipynb +++ b/notebooks/plots-worker.ipynb @@ -13,6 +13,22 @@ "import pickle" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "with open('./data-workers.pickle', 'rb') as f:\n", + " data = pickle.load(f)\n", + "\n", + "print(data)" + ] + }, { "cell_type": "code", "execution_count": null, From d69da991cf39103ad45618fe0b4849858f0a45be Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 17:26:39 +0200 Subject: [PATCH 30/43] Updated benchmark config --- dask_accelerated_worker/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index ee0ae96..5a470b4 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -12,7 +12,7 @@ # Benchmark configuration const_in_size = 4096e3 -in_sizes = [512e3, 1024e3, 2048e3, 4096e3, 8192e3] +in_sizes = [256e3, 512e3, 1024e3, 2048e3, 4096e3] batch_size = 1e3 const_batch_aggregate = 1e3 batch_aggregates = [64, 128, 256, 512, 1024, 2048, 4096, 8192] From 5be4ab67a561f594961f405f302e0fb93c1b5fc7 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 17:38:59 +0200 Subject: [PATCH 31/43] Fixed keyerror in main.py --- dask_accelerated_worker/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 5a470b4..1e4d5bc 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -108,8 +108,8 @@ def main(): if str(scheduler.workers[worker].name).split('-')[0] == 'accelerated': accelerated_workers += 1 - data[str(accelerated_workers)]['in_size'] = data_in_size - data[str(accelerated_workers)]['batch_size'] = data_batch_size + data[str(accelerated_workers) + '-accelerated']['in_size'] = data_in_size + data[str(accelerated_workers) + '-accelerated']['batch_size'] = data_batch_size # Add timestamp to data timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") From 622f23ec2b5779fd049a9b0614d1eb35d42ccfb9 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 17:50:13 +0200 Subject: [PATCH 32/43] Dont overwrite data dict and set repeats to 1 --- dask_accelerated_worker/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 1e4d5bc..c8c4f72 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -16,7 +16,7 @@ batch_size = 1e3 const_batch_aggregate = 1e3 batch_aggregates = [64, 128, 256, 512, 1024, 2048, 4096, 8192] -repeats = 5 +repeats = 1 def main(): @@ -43,9 +43,10 @@ def main(): # Make sure the desired dataset exists helpers.generate_datasets_if_needed(in_sizes, batch_size) + data = {} + # Keep running the benchmark until the user quits the client while True: - data = {} data_in_size = {} data_batch_size = {} @@ -108,8 +109,10 @@ def main(): if str(scheduler.workers[worker].name).split('-')[0] == 'accelerated': accelerated_workers += 1 - data[str(accelerated_workers) + '-accelerated']['in_size'] = data_in_size - data[str(accelerated_workers) + '-accelerated']['batch_size'] = data_batch_size + data[str(accelerated_workers)] = { + 'in_size': data_in_size, + 'batch_size': data_batch_size + } # Add timestamp to data timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") From 23ad4755a97c52465d1dbf6f00f572a47245a215 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 17:54:42 +0200 Subject: [PATCH 33/43] Set repeats to 5 --- dask_accelerated_worker/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index c8c4f72..81caa58 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -16,7 +16,7 @@ batch_size = 1e3 const_batch_aggregate = 1e3 batch_aggregates = [64, 128, 256, 512, 1024, 2048, 4096, 8192] -repeats = 1 +repeats = 5 def main(): From 75abf6614676741e2151e325acdcb8fd613dc62f Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 22:58:13 +0200 Subject: [PATCH 34/43] Made plots for both cluster benchmarks in plots-worker.ipynb --- notebooks/plots-worker.ipynb | 74 +++++++++++++++--------------------- 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/notebooks/plots-worker.ipynb b/notebooks/plots-worker.ipynb index aedcd65..9958b06 100644 --- a/notebooks/plots-worker.ipynb +++ b/notebooks/plots-worker.ipynb @@ -26,7 +26,18 @@ "with open('./data-workers.pickle', 'rb') as f:\n", " data = pickle.load(f)\n", "\n", - "print(data)" + "with open('./data-workers-0.pickle', 'rb') as f:\n", + " data['0'] = pickle.load(f)['0']\n", + "\n", + "print(data['timestamp'])\n", + "out_dir = './plots/workers/' + data['timestamp']\n", + "\n", + "# Create the directory if it does not yet exist\n", + "if not os.path.exists(out_dir):\n", + " os.makedirs(out_dir)\n", + "\n", + "copyfile('./data-workers-0.pickle', out_dir + '/data-workers-0.pickle')\n", + "copyfile('./data-workers.pickle', out_dir + '/data-workers.pickle')" ] }, { @@ -39,28 +50,10 @@ }, "outputs": [], "source": [ - "data = {}\n", - "\n", - "for i in range(1, 2):\n", - " with open('./data-' + str(i) + '.pickle', 'rb') as f:\n", - " data[i] = pickle.load(f)\n", - "\n", - "with open('./data-1-vanilla.pickle', 'rb') as f:\n", - " data_vanilla = pickle.load(f)\n", - "data_vanilla.pop('timestamp', None)\n", - "\n", - "print(data[1]['timestamp'])\n", - "out_dir = './plots/workers/' + data[1]['timestamp']\n", + "print(data)\n", "\n", - "# Create the directory if it does not yet exist\n", - "if not os.path.exists(out_dir):\n", - " os.makedirs(out_dir)\n", - "# Copy the data file to the out dir as a backup\n", - "for i in range(1, 2):\n", - " data[i].pop('timestamp', None)\n", - " copyfile('./data-' + str(i) + '.pickle', out_dir + '/data-' + str(i) + '.pickle')\n", - "\n", - "copyfile('./data-1-vanilla.pickle', out_dir + '/data-1-vanilla.pickle')" + "x_vals_in = ['256k', '512k', '1024k', '2048k', '4096k']\n", + "x_vals_batch = ['64k', '128k', '256k', '512k', '1024k', '2048k', '4096k', '8192k']" ] }, { @@ -73,25 +66,22 @@ }, "outputs": [], "source": [ - "x_values = ['1K', '2K', '4K', '8K', '16K', '32K', '64K', '128K', '256K', '512K', '1M', '2M', '4M']\n", - "\n", "fig = plt.figure(figsize=(9,7))\n", "fig.patch.set_facecolor('white')\n", "\n", - "plt.plot(x_values, list(data_vanilla.values()), color='r', marker='x', label='vanilla worker', zorder=3)\n", - "plt.plot(x_values, list(data[1].values()), color='b', marker='o', label='accelerated worker', zorder=3)\n", - "# plt.plot(x_values, list(data[2].values()), color='r', marker='^', label='2 accelerated workers', zorder=3)\n", - "# plt.plot(x_values, list(data[3].values()), color='y', marker='>', label='3 accelerated workers', zorder=3)\n", - "# plt.plot(x_values, list(data[4].values()), color='g', marker='x', label='4 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(data['0']['in_size'].values()), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(data['1']['in_size'].values()), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_in, list(data['2']['in_size'].values()), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(data['3']['in_size'].values()), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Number of records')\n", "plt.ylabel('Total query runtime (seconds)')\n", "\n", - "plt.xticks(x_values)\n", + "plt.xticks(x_vals_in)\n", "plt.xticks(rotation=-90)\n", "\n", - "plt.title('Runtime accelerated worker')\n", + "plt.title('Runtime in sizes benchmark - batch 1M')\n", "\n", "axes = plt.gca()\n", "axes.grid(which='both', axis='y', linestyle='--')\n", @@ -102,7 +92,7 @@ "plt.legend()\n", "\n", "# Save fig as pdf\n", - "plt.savefig(out_dir + '/tidre_workers_in_sizes.png')\n", + "plt.savefig(out_dir + '/cluster_in_sizes.png')\n", "\n", "plt.show()" ] @@ -117,26 +107,22 @@ }, "outputs": [], "source": [ - "x_values = ['1K', '2K', '4K', '8K', '16K', '32K', '64K', '128K', '256K', '512K', '1M', '2M', '4M']\n", - "\n", "fig = plt.figure(figsize=(9,7))\n", "fig.patch.set_facecolor('white')\n", "\n", - "speedup = np.divide(list(data_vanilla.values()), list(data[1].values()))\n", - "\n", - "plt.plot(x_values, speedup, color='b', marker='o', label='accelerated worker', zorder=3)\n", - "# plt.plot(x_values, list(data[2].values()), color='r', marker='^', label='2 accelerated workers', zorder=3)\n", - "# plt.plot(x_values, list(data[3].values()), color='y', marker='>', label='3 accelerated workers', zorder=3)\n", - "# plt.plot(x_values, list(data[4].values()), color='g', marker='x', label='4 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(data['0']['batch_size'].values()), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(data['1']['batch_size'].values()), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(data['2']['batch_size'].values()), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(data['3']['batch_size'].values()), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", - "plt.xlabel('Number of records')\n", + "plt.xlabel('Batch size')\n", "plt.ylabel('Total query runtime (seconds)')\n", "\n", - "plt.xticks(x_values)\n", + "plt.xticks(x_vals_batch)\n", "plt.xticks(rotation=-90)\n", "\n", - "plt.title('Speedup accelerated worker')\n", + "plt.title('Runtime batch sizes benchmark - in 4M')\n", "\n", "axes = plt.gca()\n", "axes.grid(which='both', axis='y', linestyle='--')\n", @@ -147,7 +133,7 @@ "plt.legend()\n", "\n", "# Save fig as pdf\n", - "plt.savefig(out_dir + '/speedup_tidre_workers_in_sizes.png')\n", + "plt.savefig(out_dir + '/cluster_batch_sizes.png')\n", "\n", "plt.show()" ] From c4dfc8f0eaf515e77507aa51753258b96b9f2a58 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 23:01:47 +0200 Subject: [PATCH 35/43] Added newline to end of utils.py --- dask_accelerated_worker/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_accelerated_worker/utils.py b/dask_accelerated_worker/utils.py index 81442f7..a0191ed 100644 --- a/dask_accelerated_worker/utils.py +++ b/dask_accelerated_worker/utils.py @@ -27,4 +27,4 @@ async def cleanup_and_stop(): signal.signal(sig, old_handlers[sig]) for sig in [signal.SIGINT, signal.SIGTERM]: - old_handlers[sig] = signal.signal(sig, handle_signal) \ No newline at end of file + old_handlers[sig] = signal.signal(sig, handle_signal) From 40e8384d353c75bff6c0baf356b45ac2bdcec5c7 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 23:13:51 +0200 Subject: [PATCH 36/43] Added option to start an RE2 accelerated worker --- dask_accelerated_worker/accelerated_worker.py | 55 ++++++++++++++++++- dask_accelerated_worker/start_worker.py | 21 ++++--- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/dask_accelerated_worker/accelerated_worker.py b/dask_accelerated_worker/accelerated_worker.py index ead9018..86d226f 100644 --- a/dask_accelerated_worker/accelerated_worker.py +++ b/dask_accelerated_worker/accelerated_worker.py @@ -9,8 +9,59 @@ logger = logging.getLogger(__name__) -# Create an accelerated worker class based on the original worker class -class AcceleratedWorker(Worker): +# Create an RE2 accelerated worker class based on the original worker class +class RE2Worker(Worker): + + def add_task( + self, + key, + function=None, + args=None, + kwargs=None, + task=worker.no_value, + who_has=None, + nbytes=None, + priority=None, + duration=None, + resource_restrictions=None, + actor=False, + **kwargs2, + ): + regex = re.compile('.*str-match.*') + if re.match(regex, key) is not None: + # This task matches the operation we want to perform on fpga + func = pickle.loads(function) + + substitute_op = CustomFilter().custom_re2 + + dsk = func.dsk + vals = dsk[func.outkey] + vals_args = vals[3] + new_vals_args = (vals_args[0], [['_func', substitute_op], vals_args[1][1]]) + new_vals = (vals[0], vals[1], vals[2], new_vals_args) + dsk[func.outkey] = new_vals + + new_func = SubgraphCallable(dsk, func.outkey, func.inkeys, "regex_callable") + function = pickle.dumps(new_func) + + super().add_task( + key, + function, + args, + kwargs, + task, + who_has, + nbytes, + priority, + duration, + resource_restrictions, + actor, + **kwargs2, + ) + + +# Create a tidre accelerated worker class based on the original worker class +class TidreWorker(Worker): def add_task( self, diff --git a/dask_accelerated_worker/start_worker.py b/dask_accelerated_worker/start_worker.py index bfe4285..40bd742 100644 --- a/dask_accelerated_worker/start_worker.py +++ b/dask_accelerated_worker/start_worker.py @@ -1,4 +1,4 @@ -from dask_accelerated_worker.accelerated_worker import AcceleratedWorker +from dask_accelerated_worker.accelerated_worker import RE2Worker, TidreWorker from dask.distributed import Worker from tornado.ioloop import IOLoop from dask_accelerated_worker.utils import install_signal_handlers @@ -14,9 +14,8 @@ parser = argparse.ArgumentParser(description='Dask Accelerated Worker.') parser.add_argument('scheduler_address', metavar='S', type=str, help='string containing the ip and port of the scheduler. Example: tcp://127.0.0.1:37983') -parser.add_argument('--accelerated', dest='accelerated', action='store_const', - const=True, default=False, - help='use the accelerated worker implementation. (default: do not use)') +parser.add_argument('type', metavar='T', type=str, + help='string containing the type of the worker. Can be `tidre`, `re2`, or `vanilla`.') args = parser.parse_args() @@ -25,12 +24,20 @@ def main(): scheduler_address = args.scheduler_address - if args.accelerated: - t = AcceleratedWorker + if args.type == 'tidre': + print('Starting Tidre worker') + t = TidreWorker worker_name = 'accelerated-' + str(time.time()) - else: + elif args.type == 're2': + print('Starting RE2 worker') + t = RE2Worker + worker_name = 'accelerated-' + str(time.time()) + elif args.type == 'vanilla': + print('Starting vanilla worker') t = Worker worker_name = 'vanilla-' + str(time.time()) + else: + raise Exception("Worker type not valid.") # Start a new worker based on the AcceleratedWorker class # This worker automatically connects to the scheduler and gets added to the worker pool From 28183cf93d9fe4ff10c081f820c142adf6f43f70 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 23:48:30 +0200 Subject: [PATCH 37/43] Cleaned up main.py and extracted functionalities to benchmarks.py and helpers.py --- dask_accelerated_worker/benchmarks.py | 95 ++++++++++++++++++++++++++ dask_accelerated_worker/helpers.py | 14 ++++ dask_accelerated_worker/main.py | 96 ++++----------------------- 3 files changed, 122 insertions(+), 83 deletions(-) create mode 100644 dask_accelerated_worker/benchmarks.py diff --git a/dask_accelerated_worker/benchmarks.py b/dask_accelerated_worker/benchmarks.py new file mode 100644 index 0000000..daee12c --- /dev/null +++ b/dask_accelerated_worker/benchmarks.py @@ -0,0 +1,95 @@ +from dask_accelerated import helpers +import time + + +def run_all_benchmarks(client, scheduler, data, benchmark_config): + + data_in_size = run_in_benchmark(client, benchmark_config) + data_batch_size = run_batch_benchmark(client, benchmark_config) + + # Count the number of accelerated workers + accelerated_workers = 0 + for worker in scheduler.workers: + if str(scheduler.workers[worker].name).split('-')[0] == 'accelerated': + accelerated_workers += 1 + + data[str(accelerated_workers)] = { + 'in_size': data_in_size, + 'batch_size': data_batch_size + } + + return data + + +def run_in_benchmark(client, benchmark_config): + + data_in_size = {} + + for in_size in benchmark_config['in_sizes']: + + lazy_result = helpers.get_lazy_result( + in_size, + benchmark_config['batch_size'], + benchmark_config['const_batch_aggregate'] + ) + + graph = lazy_result.__dask_graph__() + + # Dry run + # res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + + data_in_size[in_size] = 0 + + for i in range(benchmark_config['repeats']): + start = time.time() + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + end = time.time() + + duration = end - start + data_in_size[in_size] += duration + print( + 'In: ', in_size, + '\tBatch: ', benchmark_config['batch_size'] * benchmark_config['const_batch_aggregate'], + '\tComputed ', res, ' in ', duration, ' seconds' + ) + + data_in_size[in_size] = data_in_size[in_size] / benchmark_config['repeats'] + + return data_in_size + + +def run_batch_benchmark(client, benchmark_config): + + data_batch_size = {} + + for batch_aggregate in benchmark_config['batch_aggregates']: + + lazy_result = helpers.get_lazy_result( + benchmark_config['const_in_size'], + benchmark_config['batch_size'], + batch_aggregate + ) + + graph = lazy_result.__dask_graph__() + + # Dry run + # res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + + data_batch_size[batch_aggregate] = 0 + + for i in range(benchmark_config['repeats']): + start = time.time() + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + end = time.time() + + duration = end - start + data_batch_size[batch_aggregate] += duration + print( + 'In: ', benchmark_config['const_in_size'], + '\tBatch: ', benchmark_config['batch_size'] * batch_aggregate, + 'Computed ', res, ' in ', duration, ' seconds' + ) + + data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / benchmark_config['repeats'] + + return data_batch_size diff --git a/dask_accelerated_worker/helpers.py b/dask_accelerated_worker/helpers.py index 2b1bfb3..ae4448d 100644 --- a/dask_accelerated_worker/helpers.py +++ b/dask_accelerated_worker/helpers.py @@ -1,6 +1,8 @@ from dask.distributed import Scheduler from tornado.ioloop import IOLoop +from datetime import datetime import asyncio +import pickle def get_scheduler(): @@ -63,3 +65,15 @@ def remove_non_accelerated_workers(scheduler): ) loop.close() + + +def save_data(data): + + # Add timestamp to data + timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") + data['timestamp'] = timestamp + + # Save data to disk + data_root = '../notebooks/' + with open(data_root + 'data-workers.pickle', 'wb') as f: + pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 81caa58..0bde250 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -1,8 +1,7 @@ from dask.distributed import Client from dask_accelerated import helpers from dask_accelerated_worker import helpers as worker_helpers -import pickle -from datetime import datetime +import benchmarks import time from threading import Thread import logging @@ -10,13 +9,14 @@ logger = logging.getLogger(__name__) -# Benchmark configuration -const_in_size = 4096e3 -in_sizes = [256e3, 512e3, 1024e3, 2048e3, 4096e3] -batch_size = 1e3 -const_batch_aggregate = 1e3 -batch_aggregates = [64, 128, 256, 512, 1024, 2048, 4096, 8192] -repeats = 5 +benchmark_config = { + 'const_in_size': 4096e3, + 'in_sizes': [256e3, 512e3, 1024e3, 2048e3, 4096e3], + 'batch_size': 1e3, + 'const_batch_aggregate': 1e3, + 'batch_aggregates': [64, 128, 256, 512, 1024, 2048, 4096, 8192], + 'repeats': 5 +} def main(): @@ -41,91 +41,21 @@ def main(): input("Press Enter to perform benchmarks...") # Make sure the desired dataset exists - helpers.generate_datasets_if_needed(in_sizes, batch_size) + helpers.generate_datasets_if_needed(benchmark_config['in_sizes'], benchmark_config['batch_size']) data = {} # Keep running the benchmark until the user quits the client while True: - data_in_size = {} - data_batch_size = {} - - # In size benchmark - for in_size in in_sizes: - - lazy_result = helpers.get_lazy_result(in_size, batch_size, const_batch_aggregate) - graph = lazy_result.__dask_graph__() - - # Dry run - # res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) - - data_in_size[in_size] = 0 - - for i in range(repeats): - - start = time.time() - res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) - end = time.time() - - duration = end - start - data_in_size[in_size] += duration - print( - 'In: ', in_size, - '\tBatch: ', batch_size * const_batch_aggregate, - '\tComputed ', res, ' in ', duration, ' seconds' - ) - - data_in_size[in_size] = data_in_size[in_size] / repeats - - # Batch size benchmark - for batch_aggregate in batch_aggregates: - - lazy_result = helpers.get_lazy_result(const_in_size, batch_size, batch_aggregate) - graph = lazy_result.__dask_graph__() - - # Dry run - # res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) - - data_batch_size[batch_aggregate] = 0 - - for i in range(repeats): - start = time.time() - res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) - end = time.time() - - duration = end - start - data_batch_size[batch_aggregate] += duration - print( - 'In: ', const_in_size, - '\tBatch: ', batch_size * batch_aggregate, - 'Computed ', res, ' in ', duration, ' seconds' - ) - - data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / repeats - - # Count the number of accelerated workers - accelerated_workers = 0 - for worker in scheduler.workers: - if str(scheduler.workers[worker].name).split('-')[0] == 'accelerated': - accelerated_workers += 1 - - data[str(accelerated_workers)] = { - 'in_size': data_in_size, - 'batch_size': data_batch_size - } - - # Add timestamp to data - timestamp = datetime.now().strftime("%d-%b-%Y_%H:%M:%S") - data['timestamp'] = timestamp + data = benchmarks.run_all_benchmarks(client, scheduler, data, benchmark_config) user_input = input("Press Enter to run again or send 'q' to close the client...") if user_input == 'q': break - data_root = '../notebooks/' - with open(data_root + 'data-workers.pickle', 'wb') as f: - pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + worker_helpers.save_data(data) + # Close the client client.close() From c9b61592f6dc2a991aa86be911c57d4f13fae5a5 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sat, 8 May 2021 23:54:56 +0200 Subject: [PATCH 38/43] Added method to warm workers, effectively scattering input parquet files --- dask_accelerated_worker/benchmarks.py | 23 +++++++++++++++++++++++ dask_accelerated_worker/main.py | 2 ++ 2 files changed, 25 insertions(+) diff --git a/dask_accelerated_worker/benchmarks.py b/dask_accelerated_worker/benchmarks.py index daee12c..83ed790 100644 --- a/dask_accelerated_worker/benchmarks.py +++ b/dask_accelerated_worker/benchmarks.py @@ -2,6 +2,29 @@ import time +def warm_workers(client, scheduler, benchmark_config): + + print('Warming workers... ', end='') + + for in_size in benchmark_config['in_sizes']: + + lazy_result = helpers.get_lazy_result( + in_size, + benchmark_config['batch_size'], + in_size + ) + + graph = lazy_result.__dask_graph__() + + # Scheduler does round robin + # so we can run this for each worker in the pool + for worker in scheduler.workers: + # Dry run + res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + + print('done') + + def run_all_benchmarks(client, scheduler, data, benchmark_config): data_in_size = run_in_benchmark(client, benchmark_config) diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 0bde250..1c1e7ae 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -43,6 +43,8 @@ def main(): # Make sure the desired dataset exists helpers.generate_datasets_if_needed(benchmark_config['in_sizes'], benchmark_config['batch_size']) + benchmarks.warm_workers(client, scheduler, benchmark_config) + data = {} # Keep running the benchmark until the user quits the client From e8f1b8d12db2e477df8a9c0ea8ed370bf1a6cf27 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 9 May 2021 16:43:32 +0200 Subject: [PATCH 39/43] 6 repeats and dont consider first 3 runs in benchmark --- dask_accelerated_worker/benchmarks.py | 10 ++++++---- dask_accelerated_worker/main.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dask_accelerated_worker/benchmarks.py b/dask_accelerated_worker/benchmarks.py index 83ed790..662665c 100644 --- a/dask_accelerated_worker/benchmarks.py +++ b/dask_accelerated_worker/benchmarks.py @@ -69,14 +69,15 @@ def run_in_benchmark(client, benchmark_config): end = time.time() duration = end - start - data_in_size[in_size] += duration + if i > 2: + data_in_size[in_size] += duration print( 'In: ', in_size, '\tBatch: ', benchmark_config['batch_size'] * benchmark_config['const_batch_aggregate'], '\tComputed ', res, ' in ', duration, ' seconds' ) - data_in_size[in_size] = data_in_size[in_size] / benchmark_config['repeats'] + data_in_size[in_size] = data_in_size[in_size] / (benchmark_config['repeats'] - 3) return data_in_size @@ -106,13 +107,14 @@ def run_batch_benchmark(client, benchmark_config): end = time.time() duration = end - start - data_batch_size[batch_aggregate] += duration + if i > 2: + data_batch_size[batch_aggregate] += duration print( 'In: ', benchmark_config['const_in_size'], '\tBatch: ', benchmark_config['batch_size'] * batch_aggregate, 'Computed ', res, ' in ', duration, ' seconds' ) - data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / benchmark_config['repeats'] + data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / (benchmark_config['repeats'] - 3) return data_batch_size diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 1c1e7ae..4d1c4fa 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -15,7 +15,7 @@ 'batch_size': 1e3, 'const_batch_aggregate': 1e3, 'batch_aggregates': [64, 128, 256, 512, 1024, 2048, 4096, 8192], - 'repeats': 5 + 'repeats': 6 } From 1c05e770f782df1f8555542e3b9798cd14ca100f Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 9 May 2021 16:49:26 +0200 Subject: [PATCH 40/43] Warm workers in main loop and do consider first 3 runs --- dask_accelerated_worker/benchmarks.py | 10 ++++------ dask_accelerated_worker/main.py | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dask_accelerated_worker/benchmarks.py b/dask_accelerated_worker/benchmarks.py index 662665c..83ed790 100644 --- a/dask_accelerated_worker/benchmarks.py +++ b/dask_accelerated_worker/benchmarks.py @@ -69,15 +69,14 @@ def run_in_benchmark(client, benchmark_config): end = time.time() duration = end - start - if i > 2: - data_in_size[in_size] += duration + data_in_size[in_size] += duration print( 'In: ', in_size, '\tBatch: ', benchmark_config['batch_size'] * benchmark_config['const_batch_aggregate'], '\tComputed ', res, ' in ', duration, ' seconds' ) - data_in_size[in_size] = data_in_size[in_size] / (benchmark_config['repeats'] - 3) + data_in_size[in_size] = data_in_size[in_size] / benchmark_config['repeats'] return data_in_size @@ -107,14 +106,13 @@ def run_batch_benchmark(client, benchmark_config): end = time.time() duration = end - start - if i > 2: - data_batch_size[batch_aggregate] += duration + data_batch_size[batch_aggregate] += duration print( 'In: ', benchmark_config['const_in_size'], '\tBatch: ', benchmark_config['batch_size'] * batch_aggregate, 'Computed ', res, ' in ', duration, ' seconds' ) - data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / (benchmark_config['repeats'] - 3) + data_batch_size[batch_aggregate] = data_batch_size[batch_aggregate] / benchmark_config['repeats'] return data_batch_size diff --git a/dask_accelerated_worker/main.py b/dask_accelerated_worker/main.py index 4d1c4fa..30e9eee 100644 --- a/dask_accelerated_worker/main.py +++ b/dask_accelerated_worker/main.py @@ -43,12 +43,13 @@ def main(): # Make sure the desired dataset exists helpers.generate_datasets_if_needed(benchmark_config['in_sizes'], benchmark_config['batch_size']) - benchmarks.warm_workers(client, scheduler, benchmark_config) - data = {} # Keep running the benchmark until the user quits the client while True: + + benchmarks.warm_workers(client, scheduler, benchmark_config) + data = benchmarks.run_all_benchmarks(client, scheduler, data, benchmark_config) user_input = input("Press Enter to run again or send 'q' to close the client...") From 8cf1608567c84b2b09522e56adba5c2da7835062 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Sun, 9 May 2021 16:55:00 +0200 Subject: [PATCH 41/43] Use underscore variable for dry run result in benchmarks.py --- dask_accelerated_worker/benchmarks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_accelerated_worker/benchmarks.py b/dask_accelerated_worker/benchmarks.py index 83ed790..e843b83 100644 --- a/dask_accelerated_worker/benchmarks.py +++ b/dask_accelerated_worker/benchmarks.py @@ -20,7 +20,7 @@ def warm_workers(client, scheduler, benchmark_config): # so we can run this for each worker in the pool for worker in scheduler.workers: # Dry run - res = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) + _ = client.get(graph, (lazy_result.__dask_layers__()[0], 0)) print('done') From d3cfeba43aa1b4660db35aafe4e9929cc605ea11 Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Wed, 12 May 2021 12:05:12 +0200 Subject: [PATCH 42/43] Newest plots in plots-worker.ipynb --- notebooks/plots-worker.ipynb | 322 ++++++++++++++++++++++++++++++++++- 1 file changed, 315 insertions(+), 7 deletions(-) diff --git a/notebooks/plots-worker.ipynb b/notebooks/plots-worker.ipynb index 9958b06..963f9dd 100644 --- a/notebooks/plots-worker.ipynb +++ b/notebooks/plots-worker.ipynb @@ -26,9 +26,6 @@ "with open('./data-workers.pickle', 'rb') as f:\n", " data = pickle.load(f)\n", "\n", - "with open('./data-workers-0.pickle', 'rb') as f:\n", - " data['0'] = pickle.load(f)['0']\n", - "\n", "print(data['timestamp'])\n", "out_dir = './plots/workers/' + data['timestamp']\n", "\n", @@ -36,7 +33,6 @@ "if not os.path.exists(out_dir):\n", " os.makedirs(out_dir)\n", "\n", - "copyfile('./data-workers-0.pickle', out_dir + '/data-workers-0.pickle')\n", "copyfile('./data-workers.pickle', out_dir + '/data-workers.pickle')" ] }, @@ -53,7 +49,10 @@ "print(data)\n", "\n", "x_vals_in = ['256k', '512k', '1024k', '2048k', '4096k']\n", - "x_vals_batch = ['64k', '128k', '256k', '512k', '1024k', '2048k', '4096k', '8192k']" + "x_vals_batch = ['64k', '128k', '256k', '512k', '1024k', '2048k', '4096k', '8192k']\n", + "\n", + "m_cost_per_second = 0.4 / 60 / 60\n", + "f_cost_per_second = 1.65 / 60 / 60" ] }, { @@ -66,6 +65,8 @@ }, "outputs": [], "source": [ + "# RUNTIME IN SIZE\n", + "\n", "fig = plt.figure(figsize=(9,7))\n", "fig.patch.set_facecolor('white')\n", "\n", @@ -81,7 +82,7 @@ "plt.xticks(x_vals_in)\n", "plt.xticks(rotation=-90)\n", "\n", - "plt.title('Runtime in sizes benchmark - batch 1M')\n", + "plt.title('Runtime in sizes benchmark - batch size 1M')\n", "\n", "axes = plt.gca()\n", "axes.grid(which='both', axis='y', linestyle='--')\n", @@ -107,6 +108,161 @@ }, "outputs": [], "source": [ + "# THROUGHPUT IN SIZE\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "in_sizes = [256e3, 512e3, 1024e3, 2048e3, 4096e3]\n", + "in_bytes = np.array([x * 100 * 1 for x in in_sizes])\n", + "\n", + "throughput = {\n", + " '0': np.divide(in_bytes, list(data['0']['in_size'].values())),\n", + " '1': np.divide(in_bytes, list(data['1']['in_size'].values())),\n", + " '2': np.divide(in_bytes, list(data['2']['in_size'].values())),\n", + " '3': np.divide(in_bytes, list(data['3']['in_size'].values()))\n", + "}\n", + "\n", + "plt.plot(x_vals_in, list(throughput['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(throughput['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_in, list(throughput['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(throughput['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Number of records')\n", + "plt.ylabel('Total query throughput (bytes/s)')\n", + "\n", + "plt.xticks(x_vals_in)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('Throughput in sizes benchmark - batch size 1M')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/cluster_in_sizes_throughput.png')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# SPEEDUP IN SIZE\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "speedup = {\n", + " '0': np.divide(list(data['0']['in_size'].values()), list(data['0']['in_size'].values())),\n", + " '1': np.divide(list(data['0']['in_size'].values()), list(data['1']['in_size'].values())),\n", + " '2': np.divide(list(data['0']['in_size'].values()), list(data['2']['in_size'].values())),\n", + " '3': np.divide(list(data['0']['in_size'].values()), list(data['3']['in_size'].values()))\n", + "}\n", + "\n", + "plt.plot(x_vals_in, list(speedup['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(speedup['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_in, list(speedup['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(speedup['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Number of records')\n", + "plt.ylabel('Speedup')\n", + "\n", + "plt.xticks(x_vals_in)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('Speedup in sizes benchmark - batch size 1M')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/cluster_in_sizes_speedup.png')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# COST IN SIZE\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "cost = {\n", + " '0': np.multiply(list(data['0']['in_size'].values()), (3*m_cost_per_second)),\n", + " '1': np.multiply(list(data['1']['in_size'].values()), (2*m_cost_per_second + f_cost_per_second)),\n", + " '2': np.multiply(list(data['2']['in_size'].values()), (m_cost_per_second + 2*f_cost_per_second)),\n", + " '3': np.multiply(list(data['3']['in_size'].values()), (3*f_cost_per_second))\n", + "}\n", + "\n", + "plt.plot(x_vals_in, list(cost['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(cost['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_in, list(cost['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(cost['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Number of records')\n", + "plt.ylabel('Cost per query ($)')\n", + "\n", + "plt.xticks(x_vals_in)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('Cost in sizes benchmark - batch size 1M')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/cluster_in_sizes_cost.png')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# RUNTIME BATCH SIZE\n", + "\n", "fig = plt.figure(figsize=(9,7))\n", "fig.patch.set_facecolor('white')\n", "\n", @@ -122,7 +278,7 @@ "plt.xticks(x_vals_batch)\n", "plt.xticks(rotation=-90)\n", "\n", - "plt.title('Runtime batch sizes benchmark - in 4M')\n", + "plt.title('Runtime batch sizes benchmark - in size 4M')\n", "\n", "axes = plt.gca()\n", "axes.grid(which='both', axis='y', linestyle='--')\n", @@ -137,6 +293,158 @@ "\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# THROUGHPUT BATCH SIZE\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "in_bytes = 100 * 1 * 4096e3\n", + "\n", + "throughput = {\n", + " '0': np.divide(in_bytes, list(data['0']['batch_size'].values())),\n", + " '1': np.divide(in_bytes, list(data['1']['batch_size'].values())),\n", + " '2': np.divide(in_bytes, list(data['2']['batch_size'].values())),\n", + " '3': np.divide(in_bytes, list(data['3']['batch_size'].values()))\n", + "}\n", + "\n", + "plt.plot(x_vals_batch, list(throughput['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(throughput['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(throughput['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(throughput['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Batch size')\n", + "plt.ylabel('Total query throughput (bytes/s)')\n", + "\n", + "plt.xticks(x_vals_batch)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('Throughput batch sizes benchmark - in size 4M')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/cluster_batch_sizes_throughput.png')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# SPEEDUP BATCH SIZE\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "speedup = {\n", + " '0': np.divide(list(data['0']['batch_size'].values()), list(data['0']['batch_size'].values())),\n", + " '1': np.divide(list(data['0']['batch_size'].values()), list(data['1']['batch_size'].values())),\n", + " '2': np.divide(list(data['0']['batch_size'].values()), list(data['2']['batch_size'].values())),\n", + " '3': np.divide(list(data['0']['batch_size'].values()), list(data['3']['batch_size'].values()))\n", + "}\n", + "\n", + "plt.plot(x_vals_batch, list(speedup['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(speedup['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(speedup['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(speedup['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Batch size')\n", + "plt.ylabel('Speedup')\n", + "\n", + "plt.xticks(x_vals_batch)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('Speedup batch sizes benchmark - in size 4M')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/cluster_batch_sizes_speedup.png')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# COST BATCH SIZE\n", + "\n", + "fig = plt.figure(figsize=(9,7))\n", + "fig.patch.set_facecolor('white')\n", + "\n", + "cost = {\n", + " '0': np.multiply(list(data['0']['batch_size'].values()), (3*m_cost_per_second)),\n", + " '1': np.multiply(list(data['1']['batch_size'].values()), (2*m_cost_per_second + f_cost_per_second)),\n", + " '2': np.multiply(list(data['2']['batch_size'].values()), (m_cost_per_second + 2*f_cost_per_second)),\n", + " '3': np.multiply(list(data['3']['batch_size'].values()), (3*f_cost_per_second))\n", + "}\n", + "\n", + "plt.plot(x_vals_batch, list(cost['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(cost['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(cost['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(cost['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "\n", + "# Add xticks on the middle of the group bars\n", + "plt.xlabel('Batch size')\n", + "plt.ylabel('Cost per query ($)')\n", + "\n", + "plt.xticks(x_vals_batch)\n", + "plt.xticks(rotation=-90)\n", + "\n", + "plt.title('Cost batch sizes benchmark - in size 4M')\n", + "\n", + "axes = plt.gca()\n", + "axes.grid(which='both', axis='y', linestyle='--')\n", + "\n", + "# plt.yscale('log')\n", + "\n", + "# Create legend & Show graphic\n", + "plt.legend()\n", + "\n", + "# Save fig as pdf\n", + "plt.savefig(out_dir + '/cluster_batch_sizes_cost.png')\n", + "\n", + "plt.show()" + ] } ], "metadata": { From 702d9a22794aa218e7d1f85a3b0991ab8143c35c Mon Sep 17 00:00:00 2001 From: Bob Luppes Date: Thu, 10 Jun 2021 15:43:34 +0200 Subject: [PATCH 43/43] Open changes in plots-worker.ipynb --- notebooks/plots-worker.ipynb | 100 +++++++++++++++++------------------ 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/notebooks/plots-worker.ipynb b/notebooks/plots-worker.ipynb index 963f9dd..181beaa 100644 --- a/notebooks/plots-worker.ipynb +++ b/notebooks/plots-worker.ipynb @@ -70,10 +70,10 @@ "fig = plt.figure(figsize=(9,7))\n", "fig.patch.set_facecolor('white')\n", "\n", - "plt.plot(x_vals_in, list(data['0']['in_size'].values()), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(data['1']['in_size'].values()), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_in, list(data['2']['in_size'].values()), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(data['3']['in_size'].values()), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(data['0']['in_size'].values()), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_in, list(data['1']['in_size'].values()), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_in, list(data['2']['in_size'].values()), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_in, list(data['3']['in_size'].values()), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Number of records')\n", @@ -118,15 +118,15 @@ "\n", "throughput = {\n", " '0': np.divide(in_bytes, list(data['0']['in_size'].values())),\n", - " '1': np.divide(in_bytes, list(data['1']['in_size'].values())),\n", - " '2': np.divide(in_bytes, list(data['2']['in_size'].values())),\n", - " '3': np.divide(in_bytes, list(data['3']['in_size'].values()))\n", + " '1': np.divide(in_bytes, list(data['1']['in_size'].values()))\n", + " # '2': np.divide(in_bytes, list(data['2']['in_size'].values())),\n", + " # '3': np.divide(in_bytes, list(data['3']['in_size'].values()))\n", "}\n", "\n", - "plt.plot(x_vals_in, list(throughput['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(throughput['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_in, list(throughput['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(throughput['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(throughput['0']), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_in, list(throughput['1']), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_in, list(throughput['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_in, list(throughput['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Number of records')\n", @@ -168,15 +168,15 @@ "\n", "speedup = {\n", " '0': np.divide(list(data['0']['in_size'].values()), list(data['0']['in_size'].values())),\n", - " '1': np.divide(list(data['0']['in_size'].values()), list(data['1']['in_size'].values())),\n", - " '2': np.divide(list(data['0']['in_size'].values()), list(data['2']['in_size'].values())),\n", - " '3': np.divide(list(data['0']['in_size'].values()), list(data['3']['in_size'].values()))\n", + " '1': np.divide(list(data['0']['in_size'].values()), list(data['1']['in_size'].values()))\n", + " # '2': np.divide(list(data['0']['in_size'].values()), list(data['2']['in_size'].values())),\n", + " # '3': np.divide(list(data['0']['in_size'].values()), list(data['3']['in_size'].values()))\n", "}\n", "\n", - "plt.plot(x_vals_in, list(speedup['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(speedup['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_in, list(speedup['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(speedup['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(speedup['0']), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_in, list(speedup['1']), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_in, list(speedup['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_in, list(speedup['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Number of records')\n", @@ -217,16 +217,16 @@ "fig.patch.set_facecolor('white')\n", "\n", "cost = {\n", - " '0': np.multiply(list(data['0']['in_size'].values()), (3*m_cost_per_second)),\n", - " '1': np.multiply(list(data['1']['in_size'].values()), (2*m_cost_per_second + f_cost_per_second)),\n", - " '2': np.multiply(list(data['2']['in_size'].values()), (m_cost_per_second + 2*f_cost_per_second)),\n", - " '3': np.multiply(list(data['3']['in_size'].values()), (3*f_cost_per_second))\n", + " '0': np.multiply(list(data['0']['in_size'].values()), (m_cost_per_second)),\n", + " '1': np.multiply(list(data['1']['in_size'].values()), (f_cost_per_second)),\n", + " # '2': np.multiply(list(data['2']['in_size'].values()), (m_cost_per_second + 2*f_cost_per_second)),\n", + " # '3': np.multiply(list(data['3']['in_size'].values()), (3*f_cost_per_second))\n", "}\n", "\n", - "plt.plot(x_vals_in, list(cost['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(cost['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_in, list(cost['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_in, list(cost['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_in, list(cost['0']), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_in, list(cost['1']), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_in, list(cost['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_in, list(cost['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Number of records')\n", @@ -266,10 +266,10 @@ "fig = plt.figure(figsize=(9,7))\n", "fig.patch.set_facecolor('white')\n", "\n", - "plt.plot(x_vals_batch, list(data['0']['batch_size'].values()), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(data['1']['batch_size'].values()), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_batch, list(data['2']['batch_size'].values()), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(data['3']['batch_size'].values()), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(data['0']['batch_size'].values()), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(data['1']['batch_size'].values()), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_batch, list(data['2']['batch_size'].values()), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_batch, list(data['3']['batch_size'].values()), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Batch size')\n", @@ -314,14 +314,14 @@ "throughput = {\n", " '0': np.divide(in_bytes, list(data['0']['batch_size'].values())),\n", " '1': np.divide(in_bytes, list(data['1']['batch_size'].values())),\n", - " '2': np.divide(in_bytes, list(data['2']['batch_size'].values())),\n", - " '3': np.divide(in_bytes, list(data['3']['batch_size'].values()))\n", + " # '2': np.divide(in_bytes, list(data['2']['batch_size'].values())),\n", + " # '3': np.divide(in_bytes, list(data['3']['batch_size'].values()))\n", "}\n", "\n", - "plt.plot(x_vals_batch, list(throughput['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(throughput['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_batch, list(throughput['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(throughput['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(throughput['0']), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(throughput['1']), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_batch, list(throughput['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_batch, list(throughput['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Batch size')\n", @@ -364,14 +364,14 @@ "speedup = {\n", " '0': np.divide(list(data['0']['batch_size'].values()), list(data['0']['batch_size'].values())),\n", " '1': np.divide(list(data['0']['batch_size'].values()), list(data['1']['batch_size'].values())),\n", - " '2': np.divide(list(data['0']['batch_size'].values()), list(data['2']['batch_size'].values())),\n", - " '3': np.divide(list(data['0']['batch_size'].values()), list(data['3']['batch_size'].values()))\n", + " # '2': np.divide(list(data['0']['batch_size'].values()), list(data['2']['batch_size'].values())),\n", + " # '3': np.divide(list(data['0']['batch_size'].values()), list(data['3']['batch_size'].values()))\n", "}\n", "\n", - "plt.plot(x_vals_batch, list(speedup['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(speedup['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_batch, list(speedup['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(speedup['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(speedup['0']), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(speedup['1']), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_batch, list(speedup['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_batch, list(speedup['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Batch size')\n", @@ -412,16 +412,16 @@ "fig.patch.set_facecolor('white')\n", "\n", "cost = {\n", - " '0': np.multiply(list(data['0']['batch_size'].values()), (3*m_cost_per_second)),\n", - " '1': np.multiply(list(data['1']['batch_size'].values()), (2*m_cost_per_second + f_cost_per_second)),\n", - " '2': np.multiply(list(data['2']['batch_size'].values()), (m_cost_per_second + 2*f_cost_per_second)),\n", - " '3': np.multiply(list(data['3']['batch_size'].values()), (3*f_cost_per_second))\n", + " '0': np.multiply(list(data['0']['batch_size'].values()), (m_cost_per_second)),\n", + " '1': np.multiply(list(data['1']['batch_size'].values()), (f_cost_per_second)),\n", + " # '2': np.multiply(list(data['2']['batch_size'].values()), (m_cost_per_second + 2*f_cost_per_second)),\n", + " # '3': np.multiply(list(data['3']['batch_size'].values()), (3*f_cost_per_second))\n", "}\n", "\n", - "plt.plot(x_vals_batch, list(cost['0']), color='r', marker='x', label='0 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(cost['1']), color='g', marker='o', label='1 accelerated worker', zorder=3)\n", - "plt.plot(x_vals_batch, list(cost['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", - "plt.plot(x_vals_batch, list(cost['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", + "plt.plot(x_vals_batch, list(cost['0']), color='r', marker='x', label='vanilla worker', zorder=3)\n", + "plt.plot(x_vals_batch, list(cost['1']), color='g', marker='o', label='tidre worker', zorder=3)\n", + "# plt.plot(x_vals_batch, list(cost['2']), color='b', marker='>', label='2 accelerated workers', zorder=3)\n", + "# plt.plot(x_vals_batch, list(cost['3']), color='orange', marker='^', label='3 accelerated workers', zorder=3)\n", "\n", "# Add xticks on the middle of the group bars\n", "plt.xlabel('Batch size')\n",