From 99a5c480c28cfe3eb19a1b428e591192e02a0379 Mon Sep 17 00:00:00 2001 From: Chris Ostrouchov Date: Fri, 9 Dec 2022 10:26:25 -0500 Subject: [PATCH] Rename package to fs-queue --- README.md | 16 ++++++++++------ file_queue/__init__.py | 1 - fs_queue/__init__.py | 1 + {file_queue => fs_queue}/__main__.py | 2 +- {file_queue => fs_queue}/cli.py | 0 {file_queue => fs_queue}/core.py | 2 +- {file_queue => fs_queue}/plugins/__init__.py | 0 {file_queue => fs_queue}/plugins/dask.py | 2 +- {file_queue => fs_queue}/plugins/filelock.py | 2 +- {file_queue => fs_queue}/plugins/ssh.py | 2 +- {file_queue => fs_queue}/utils.py | 0 pyproject.toml | 6 +++--- tests/test_integration.py | 2 +- tests/test_unit.py | 2 +- 14 files changed, 21 insertions(+), 17 deletions(-) delete mode 100644 file_queue/__init__.py create mode 100644 fs_queue/__init__.py rename {file_queue => fs_queue}/__main__.py (67%) rename {file_queue => fs_queue}/cli.py (100%) rename {file_queue => fs_queue}/core.py (99%) rename {file_queue => fs_queue}/plugins/__init__.py (100%) rename {file_queue => fs_queue}/plugins/dask.py (95%) rename {file_queue => fs_queue}/plugins/filelock.py (86%) rename {file_queue => fs_queue}/plugins/ssh.py (98%) rename {file_queue => fs_queue}/utils.py (100%) diff --git a/README.md b/README.md index 9b5bb8c..7435e4a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# File Queue +# Filesystem Task Queue A task queue using the filesystem as the message queue. This project was motivated by the use case where it is hard or near impossible to @@ -27,13 +27,17 @@ chunking them or using plugins like breaking it into many small tasks). Each task state modifications results in 2-4 IOPS on the filesystem. +# Install + + - `pip install fs-queue` + # API Creating a queue is as simple as supplying a directory where the queue will reside. ```python -from file_queue import Queue +from fs_queue import Queue queue = Queue("path/to/queue") ``` @@ -42,7 +46,7 @@ Submitting jobs and monitoring over SSH is also supported via the same interface. Workers currently cannot connect over SSH. ```python -from file_queue.plugins import SSHQueue +from fs_queue.plugins import SSHQueue queue = SSHQueue("ssh://:@:/") ``` @@ -75,10 +79,10 @@ Starting a worker is as simple as giving a filesystem directory where the queue will reside. ```shell -file-queue-worker --path ./path/to/queue +fs-queue-worker --path ./path/to/queue ``` -A `dask` worker is supported via `file_queue.plugin.dask.DaskWorker` +A `dask` worker is supported via `fs_queue.plugin.dask.DaskWorker` for sending jobs to the dask cluster instead of executing locally. A worker runs a continuous loop gathering tasks in the task queue. The @@ -89,4 +93,4 @@ will: # License -[BSD-3]() +[BSD-3](./LICENSE) diff --git a/file_queue/__init__.py b/file_queue/__init__.py deleted file mode 100644 index 471b6d0..0000000 --- a/file_queue/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from file_queue.core import Queue, Job, Worker, JobStatus # noqa diff --git a/fs_queue/__init__.py b/fs_queue/__init__.py new file mode 100644 index 0000000..86ec11b --- /dev/null +++ b/fs_queue/__init__.py @@ -0,0 +1 @@ +from fs_queue.core import Queue, Job, Worker, JobStatus # noqa diff --git a/file_queue/__main__.py b/fs_queue/__main__.py similarity index 67% rename from file_queue/__main__.py rename to fs_queue/__main__.py index df96450..e949179 100644 --- a/file_queue/__main__.py +++ b/fs_queue/__main__.py @@ -1,4 +1,4 @@ -from file_queue.cli import cli +from fs_queue.cli import cli def main(): diff --git a/file_queue/cli.py b/fs_queue/cli.py similarity index 100% rename from file_queue/cli.py rename to fs_queue/cli.py diff --git a/file_queue/core.py b/fs_queue/core.py similarity index 99% rename from file_queue/core.py rename to fs_queue/core.py index e860489..67c092f 100644 --- a/file_queue/core.py +++ b/fs_queue/core.py @@ -10,7 +10,7 @@ import time from typing import Union -from file_queue import utils +from fs_queue import utils logger = logging.getLogger(__name__) diff --git a/file_queue/plugins/__init__.py b/fs_queue/plugins/__init__.py similarity index 100% rename from file_queue/plugins/__init__.py rename to fs_queue/plugins/__init__.py diff --git a/file_queue/plugins/dask.py b/fs_queue/plugins/dask.py similarity index 95% rename from file_queue/plugins/dask.py rename to fs_queue/plugins/dask.py index a042d3c..50d95d8 100644 --- a/file_queue/plugins/dask.py +++ b/fs_queue/plugins/dask.py @@ -1,6 +1,6 @@ import logging -from file_queue.core import Worker, Queue, Job +from fs_queue.core import Worker, Queue, Job import distributed diff --git a/file_queue/plugins/filelock.py b/fs_queue/plugins/filelock.py similarity index 86% rename from file_queue/plugins/filelock.py rename to fs_queue/plugins/filelock.py index 9c88f21..f8116c9 100644 --- a/file_queue/plugins/filelock.py +++ b/fs_queue/plugins/filelock.py @@ -4,7 +4,7 @@ import filelock -from file_queue.core import DummyLock +from fs_queue.core import DummyLock class FileLock(DummyLock): diff --git a/file_queue/plugins/ssh.py b/fs_queue/plugins/ssh.py similarity index 98% rename from file_queue/plugins/ssh.py rename to fs_queue/plugins/ssh.py index f49135d..6628f49 100644 --- a/file_queue/plugins/ssh.py +++ b/fs_queue/plugins/ssh.py @@ -7,7 +7,7 @@ import paramiko from paramiko.client import SSHClient -from file_queue.core import Queue, JSONSerializer, DummyLock, Job, JobStatus +from fs_queue.core import Queue, JSONSerializer, DummyLock, Job, JobStatus class SSHJob(Job): diff --git a/file_queue/utils.py b/fs_queue/utils.py similarity index 100% rename from file_queue/utils.py rename to fs_queue/utils.py diff --git a/pyproject.toml b/pyproject.toml index 02f1951..525f674 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools", "setuptools-scm"] build-backend = "setuptools.build_meta" [project] -name = "file-queue" -description = "File based task queue" +name = "fs-queue" +description = "Filesystem based task queue" readme = "README.md" requires-python = ">=3.7" license = {text = "BSD 3-Clause License"} @@ -18,4 +18,4 @@ dev = ["pytest", "black", "flake8", "build", "twine"] extra = ["filelock", "paramiko", "dask", "distributed"] [project.scripts] -file-queue-worker = "file_queue.__main__:main" \ No newline at end of file +fs-queue-worker = "fs_queue.__main__:main" \ No newline at end of file diff --git a/tests/test_integration.py b/tests/test_integration.py index 04e1e89..6089608 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,5 +1,5 @@ import file_queue -from file_queue.plugins.ssh import SSHQueue +from fs_queue.plugins.ssh import SSHQueue import operator diff --git a/tests/test_unit.py b/tests/test_unit.py index fdc79be..3b3b840 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -1,4 +1,4 @@ -from file_queue import Queue, JobStatus +from fs_queue import Queue, JobStatus def add(a: int, b: int):