From 2b24e08ce19dada6b8b6d2303671e829e27aac9a Mon Sep 17 00:00:00 2001 From: Ioannis Paraskevakos Date: Tue, 15 Sep 2026 19:05:51 +0300 Subject: [PATCH] ferat:docs --- .gitignore | 1 + .readthedocs.yml | 11 ++++ docs/Makefile | 20 +++++++ docs/make.bat | 35 ++++++++++++ docs/source/Configuration.rst | 104 ++++++++++++++++++++++++++++++++++ docs/source/Deployment.rst | 96 +++++++++++++++++++++++++++++++ docs/source/Librarians.rst | 83 +++++++++++++++++++++++++++ docs/source/Operations.rst | 47 +++++++++++++++ docs/source/conf.py | 13 +++++ docs/source/index.rst | 28 +++++++++ 10 files changed, 438 insertions(+) create mode 100644 .readthedocs.yml create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/Configuration.rst create mode 100644 docs/source/Deployment.rst create mode 100644 docs/source/Librarians.rst create mode 100644 docs/source/Operations.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst diff --git a/.gitignore b/.gitignore index 15779c2..95a2662 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,7 @@ instance/ # Sphinx documentation docs/_build/ +docs/build/ # PyBuilder .pybuilder/ diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..2c0f25b --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,11 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/source/conf.py diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/Configuration.rst b/docs/source/Configuration.rst new file mode 100644 index 0000000..234abc4 --- /dev/null +++ b/docs/source/Configuration.rst @@ -0,0 +1,104 @@ +Configuration +============= + +The archivist is configured with a single JSON file, passed with ``-c`` +(``archivist -c config.json start-server``) or through the ``ARCHIVIST_CONFIG_PATH`` +environment variable. In the container image the file is expected at +``/etc/archivist/config.json``. Settings are read at startup, so restart the +archivist after changing them. + +The configuration variables are as follows: + +- ``name``: The name of this archivist. It is sent with every callback, and must + match the ``--name`` the archivist is registered under on the librarian. +- ``debug``: Enables the API docs. Set to ``false`` in production. +- ``log_level``: The log level. We recommend ``INFO`` for production. +- ``host``: The host to bind to. Set to ``0.0.0.0`` inside a container. +- ``port``: The port to bind to (default ``8080``). +- ``displayed_site_name``, ``displayed_site_description``: Shown by the health + endpoint. +- Database settings: + + * ``database_driver``: The SQLAlchemy driver, e.g. ``postgresql``. + * ``database_host``, ``database_port``, ``database``: Where the database is. + * ``database_user``: The database user. + * ``database_password_file``: File containing the database password. Use + ``database_password`` only for local testing. +- Storage settings: + + * ``archive_type``: The storage backend. Only ``posix`` is implemented. + * ``local_root``: The common root of the librarian's files. Must be a parent of + every ``instance_path`` the librarian sends. + * ``archive_root``: Where archived files are written. A file at + ``local_root/a/b.txt`` is archived to ``archive_root/a/b.txt``. + * ``maximal_size_bytes``: The largest manifest accepted, in bytes. Must be at + least the librarian's ``filesize_per_run``. +- Throughput settings: + + * ``storage_threads``: Archive jobs copied at the same time. We recommend ``1``. + * ``copy_threads``: Files copied at the same time within one job. Use ``8`` for + network storage and ``1`` for local disk. Total concurrent copies are + ``storage_threads × copy_threads``. + * ``copy_attempts``: Attempts per file before the copy counts as failed. + * ``copy_retry_base_seconds``: Delay before the first retry, doubling after each. + * ``worker_poll_interval_seconds``: How often idle workers check for new work. + * ``max_archive_retries``: Times an interrupted job is retried on restart before + it is marked failed. +- Librarian settings (see :doc:`Librarians`): + + * ``clients``: Librarians allowed to submit manifests, and their credentials. + * ``librarians``: Where to send callbacks, and the credentials to use. + * ``require_client_auth``: Reject manifests without valid credentials (default + ``true``). + * ``cli_librarian_name``: The name used for jobs submitted with the command + line. These never send a callback. +- Callback settings: + + * ``callback_max_attempts``: Attempts before a callback is marked failed. + * ``callback_retry_base_seconds``: Delay before the first retry, doubling after each. + * ``callback_poll_interval_seconds``: How often the callback worker checks for work. + * ``callback_timeout_seconds``: Timeout for each callback request. + +Store every password in a file and point to it with a ``password_file`` setting. + +An example configuration: + +.. code-block:: json + + { + "name": "archivist-princeton", + "host": "0.0.0.0", + "port": 8080, + "log_level": "INFO", + + "database_driver": "postgresql", + "database_host": "archivist-database", + "database_port": 5432, + "database": "archivist", + "database_user": "archivist", + "database_password_file": "/run/secrets/archivist_db_password", + + "archive_type": "posix", + "local_root": "/gpfs/SIMONSOBS/so/tracked", + "archive_root": "/tigerdata/so/archive", + "maximal_size_bytes": 549755813888, + + "storage_threads": 1, + "copy_threads": 8, + "worker_poll_interval_seconds": 30.0, + "max_archive_retries": 20, + + "librarians": { + "princeton_librarian": { + "url": "http://librarian:21109", + "username": "archuser", + "password_file": "/run/secrets/archivist_authenticator" + } + }, + "clients": { + "princeton_librarian": { + "username": "archuser", + "password_file": "/run/secrets/archivist_authenticator" + } + } + } diff --git a/docs/source/Deployment.rst b/docs/source/Deployment.rst new file mode 100644 index 0000000..8d39d80 --- /dev/null +++ b/docs/source/Deployment.rst @@ -0,0 +1,96 @@ +Deployment +========== + +We recommend deploying the archivist with containers, on the same compose network +as the librarian it serves. You will need three containers: + +1. A PostgreSQL database for the archivist (separate from the librarian's). +2. A one-shot migration container that creates or upgrades the database schema. +3. The archivist server. + +The archivist does not create its schema on startup. Run the migrations when +the database is first created, and again after any upgrade that adds them; +ordinary restarts do not need them. + +Mounts +------ + +The archivist container needs three mounts: + +- The configuration directory at ``/etc/archivist``, containing ``config.json``. +- The librarian's store, **at the same path** the librarian uses. The archivist + reads each file at the ``instance_path`` given in the manifest. Mount it + read-only. +- The archive storage, at the ``archive_root`` in the configuration. + +The archivist creates any directories it needs below ``archive_root``. + +Compose Example +--------------- + +.. code-block:: yaml + + services: + archivist-database: + image: postgres:16 + restart: always + volumes: + - /data/archivist_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: "archivist" + POSTGRES_DB: "archivist" + POSTGRES_PASSWORD_FILE: /run/secrets/archivist_db_password + secrets: [archivist_db_password] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U archivist -d archivist"] + interval: 10s + retries: 5 + + archivist-migrate: + image: simonsobs/archivist:v0.1.0 + restart: "no" + entrypoint: ["alembic", "-c", "/archivist/alembic.ini", "upgrade", "head"] + volumes: + - /data/archivist/config:/etc/archivist + secrets: [archivist_db_password, archivist_authenticator] + depends_on: + archivist-database: + condition: service_healthy + + archivist: + image: simonsobs/archivist:v0.1.0 + restart: always + volumes: + - /data/archivist/config:/etc/archivist + - /gpfs/SIMONSOBS/so/tracked:/gpfs/SIMONSOBS/so/tracked:ro + - /tigerdata/so/archive:/tigerdata/so/archive + secrets: [archivist_db_password, archivist_authenticator] + depends_on: + archivist-migrate: + condition: service_completed_successfully + + secrets: + archivist_db_password: + file: /data/archivist/secrets/db_password + archivist_authenticator: + file: /data/archivist/secrets/authenticator_password + +The migration container needs the secrets too, because loading the configuration +reads every password file. + +Migrations +---------- + +To run the migrations by hand: + +.. code-block:: bash + + podman compose run --rm archivist-migrate + +A database created before the archivist used migrations already has its tables but +no migration record. Mark it as up to date once, before the first upgrade: + +.. code-block:: bash + + podman compose run --rm --entrypoint alembic archivist-migrate \ + -c /archivist/alembic.ini stamp head diff --git a/docs/source/Librarians.rst b/docs/source/Librarians.rst new file mode 100644 index 0000000..1cdd8c9 --- /dev/null +++ b/docs/source/Librarians.rst @@ -0,0 +1,83 @@ +Registering a Librarian +======================= + +A librarian and an archivist talk in both directions: the librarian submits +manifests, and the archivist calls back when an archive is stored. Both directions +use HTTP Basic with the **same** ``user:password`` pair, so there is one credential +to provision. + +Registering a librarian takes three steps: generate the credential, configure the +archivist, and register the archivist on the librarian. + +Generate the Credential +----------------------- + +Write a random password to a file readable by both services: + +.. code-block:: bash + + python3 -c 'import secrets; print(secrets.token_urlsafe(32), end="")' > authenticator_password + +Avoid ``:`` in the password; the librarian splits the authenticator on it. + +Configure the Archivist +----------------------- + +Add the librarian under both ``clients`` and ``librarians`` in the +:doc:`configuration `, keyed by the librarian's own ``name`` +(from its ``server_settings.json``): + +.. code-block:: json + + { + "clients": { + "princeton_librarian": { + "username": "archuser", + "password_file": "/run/secrets/archivist_authenticator" + } + }, + "librarians": { + "princeton_librarian": { + "url": "http://librarian:21109", + "username": "archuser", + "password_file": "/run/secrets/archivist_authenticator" + } + } + } + +- ``clients`` lets the librarian submit manifests. The archivist rejects a manifest + whose ``librarian_name`` does not match the entry the credentials belong to. +- ``librarians`` tells the archivist where to send callbacks. ``url`` is the + librarian's base URL; the callback path is added to it. +- ``username`` is a free choice and is not the librarian's name. + +Restart the archivist to load the change. + +Register on the Librarian +------------------------- + +On the librarian, register the archivist with the same credential: + +.. code-block:: bash + + librarian add-archivist $LIBRARIAN_NAME --name=archivist-princeton + --url=http://archivist + --port=8080 + --authenticator=archuser:$password + +Then add a ``create_archive`` background task that sends manifests to it. See the +librarian documentation on +`connecting an archivist `_ +and `background tasks `_. + +Names That Must Match +--------------------- + +Most registration problems are a mismatch in one of these: + +- The librarian's ``name`` is the key under ``clients`` and ``librarians``. +- The archivist's ``name`` is the ``--name`` given to ``add-archivist``, and the + ``archivist_name`` of the ``create_archive`` task. +- The ``username`` and password are the two halves of ``--authenticator``. +- The task's ``filesize_per_run`` is at most the archivist's ``maximal_size_bytes``. + A larger manifest is rejected, and the same files are selected again on every run. diff --git a/docs/source/Operations.rst b/docs/source/Operations.rst new file mode 100644 index 0000000..84e34df --- /dev/null +++ b/docs/source/Operations.rst @@ -0,0 +1,47 @@ +Operations +========== + +Command Line +------------ + +Every command takes the configuration file with ``-c``. Inside the container, run +them with ``podman exec archivist archivist -c /etc/archivist/config.json ...``. + +- ``start-server``: Starts the HTTP server and background workers. This is the + container's default command. +- ``archive --source-path $PATH``: Submits a file or directory under ``local_root`` + to the running server, without a librarian. Needs a ``__cli__`` entry under + ``clients``. No callback is sent. +- ``requeue-archive --manifest-id $ID``: Stores an archive again. Use it for an + archive that failed, or completed without all of its files. Files already + present at the right size are skipped. +- ``resend-callback --manifest-id $ID``: Sends the callback for a completed + archive again. Without ``--manifest-id``, resends every failed callback. + +Archive Jobs +------------ + +Each manifest becomes one archive job. Jobs are copied in the order they arrive. +When a job finishes, it is marked completed or failed, and a completed job's +callback is sent to its librarian. + +If the archivist restarts during a copy, the job is checked on startup: if every +file is present it is marked completed, otherwise it is queued again, up to +``max_archive_retries`` times. + +A job fails when any file still cannot be copied after ``copy_attempts``. The error +is logged as ``Archive failed to store``. No callback is sent for a +failed job; fix the cause and run ``requeue-archive``. + +Checking State +-------------- + +To see the state of recent jobs: + +.. code-block:: bash + + podman exec archivist-database psql -U archivist -d archivist -c \ + "SELECT manifest_id, completed, failed, callback_state, callback_last_error + FROM archive ORDER BY created_time DESC LIMIT 20;" + +The archivist also serves a health endpoint at ``/health``. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..d21e71f --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,13 @@ +# Configuration file for the Sphinx documentation builder. +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +project = "Archivist" +copyright = "2025-2026, Simons Observatory" +author = "Simons Observatory" + +extensions = [] +templates_path = ["_templates"] +root_doc = "index" +exclude_patterns = [] + +html_theme = "alabaster" diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..525cec8 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,28 @@ +Welcome to Archivist's documentation! +===================================== + +The archivist is a service for long-term storage of data tracked by a +`librarian `_. A librarian sends the archivist +a manifest describing a batch of files; the archivist copies those files from the +librarian's stores into archive storage, and calls the librarian back when they +are stored. File data is never sent over HTTP: the archivist reads it in place. + +The archivist is made up of four parts: + +- The HTTP server, a FastAPI server that accepts manifests from librarians. +- The database, a PostgreSQL database that records manifests and the state of + each archive job. +- The background workers, which copy files into archive storage and send + callbacks to librarians. +- The command-line client, used to start the server and manage archive jobs. + +Indices and tables +================== + +.. toctree:: + :maxdepth: 2 + + Deployment + Configuration + Librarians + Operations