ferat:docs

.gitignore

@@ -73,6 +73,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/build/
 
 # PyBuilder
 .pybuilder/

.readthedocs.yml

@@ -0,0 +1,11 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: docs/source/conf.py

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/Configuration.rst

@@ -0,0 +1,104 @@
+Configuration
+=============
+
+The archivist is configured with a single JSON file, passed with ``-c``
+(``archivist -c config.json start-server``) or through the ``ARCHIVIST_CONFIG_PATH``
+environment variable. In the container image the file is expected at
+``/etc/archivist/config.json``. Settings are read at startup, so restart the
+archivist after changing them.
+
+The configuration variables are as follows:
+
+- ``name``: The name of this archivist. It is sent with every callback, and must
+  match the ``--name`` the archivist is registered under on the librarian.
+- ``debug``: Enables the API docs. Set to ``false`` in production.
+- ``log_level``: The log level. We recommend ``INFO`` for production.
+- ``host``: The host to bind to. Set to ``0.0.0.0`` inside a container.
+- ``port``: The port to bind to (default ``8080``).
+- ``displayed_site_name``, ``displayed_site_description``: Shown by the health
+  endpoint.
+- Database settings:
+
+  * ``database_driver``: The SQLAlchemy driver, e.g. ``postgresql``.
+  * ``database_host``, ``database_port``, ``database``: Where the database is.
+  * ``database_user``: The database user.
+  * ``database_password_file``: File containing the database password. Use
+    ``database_password`` only for local testing.
+- Storage settings:
+
+  * ``archive_type``: The storage backend. Only ``posix`` is implemented.
+  * ``local_root``: The common root of the librarian's files. Must be a parent of
+    every ``instance_path`` the librarian sends.
+  * ``archive_root``: Where archived files are written. A file at
+    ``local_root/a/b.txt`` is archived to ``archive_root/a/b.txt``.
+  * ``maximal_size_bytes``: The largest manifest accepted, in bytes. Must be at
+    least the librarian's ``filesize_per_run``.
+- Throughput settings:
+
+  * ``storage_threads``: Archive jobs copied at the same time. We recommend ``1``.
+  * ``copy_threads``: Files copied at the same time within one job. Use ``8`` for
+    network storage and ``1`` for local disk. Total concurrent copies are
+    ``storage_threads × copy_threads``.
+  * ``copy_attempts``: Attempts per file before the copy counts as failed.
+  * ``copy_retry_base_seconds``: Delay before the first retry, doubling after each.
+  * ``worker_poll_interval_seconds``: How often idle workers check for new work.
+  * ``max_archive_retries``: Times an interrupted job is retried on restart before
+    it is marked failed.
+- Librarian settings (see :doc:`Librarians`):
+
+  * ``clients``: Librarians allowed to submit manifests, and their credentials.
+  * ``librarians``: Where to send callbacks, and the credentials to use.
+  * ``require_client_auth``: Reject manifests without valid credentials (default
+    ``true``).
+  * ``cli_librarian_name``: The name used for jobs submitted with the command
+    line. These never send a callback.
+- Callback settings:
+
+  * ``callback_max_attempts``: Attempts before a callback is marked failed.
+  * ``callback_retry_base_seconds``: Delay before the first retry, doubling after each.
+  * ``callback_poll_interval_seconds``: How often the callback worker checks for work.
+  * ``callback_timeout_seconds``: Timeout for each callback request.
+
+Store every password in a file and point to it with a ``password_file`` setting.
+
+An example configuration:
+
+.. code-block:: json
+
+    {
+      "name": "archivist-princeton",
+      "host": "0.0.0.0",
+      "port": 8080,
+      "log_level": "INFO",
+
+      "database_driver": "postgresql",
+      "database_host": "archivist-database",
+      "database_port": 5432,
+      "database": "archivist",
+      "database_user": "archivist",
+      "database_password_file": "/run/secrets/archivist_db_password",
+
+      "archive_type": "posix",
+      "local_root": "/gpfs/SIMONSOBS/so/tracked",
+      "archive_root": "/tigerdata/so/archive",
+      "maximal_size_bytes": 549755813888,
+
+      "storage_threads": 1,
+      "copy_threads": 8,
+      "worker_poll_interval_seconds": 30.0,
+      "max_archive_retries": 20,
+
+      "librarians": {
+        "princeton_librarian": {
+          "url": "http://librarian:21109",
+          "username": "archuser",
+          "password_file": "/run/secrets/archivist_authenticator"
+        }
+      },
+      "clients": {
+        "princeton_librarian": {
+          "username": "archuser",
+          "password_file": "/run/secrets/archivist_authenticator"
+        }
+      }
+    }

docs/source/Deployment.rst

@@ -0,0 +1,96 @@
+Deployment
+==========
+
+We recommend deploying the archivist with containers, on the same compose network
+as the librarian it serves. You will need three containers:
+
+1. A PostgreSQL database for the archivist (separate from the librarian's).
+2. A one-shot migration container that creates or upgrades the database schema.
+3. The archivist server.
+
+The archivist does not create its schema on startup. Run the migrations when
+the database is first created, and again after any upgrade that adds them;
+ordinary restarts do not need them.
+
+Mounts
+------
+
+The archivist container needs three mounts:
+
+- The configuration directory at ``/etc/archivist``, containing ``config.json``.
+- The librarian's store, **at the same path** the librarian uses. The archivist
+  reads each file at the ``instance_path`` given in the manifest. Mount it
+  read-only.
+- The archive storage, at the ``archive_root`` in the configuration.
+
+The archivist creates any directories it needs below ``archive_root``.
+
+Compose Example
+---------------
+
+.. code-block:: yaml
+
+    services:
+      archivist-database:
+        image: postgres:16
+        restart: always
+        volumes:
+          - /data/archivist_data:/var/lib/postgresql/data
+        environment:
+          POSTGRES_USER: "archivist"
+          POSTGRES_DB: "archivist"
+          POSTGRES_PASSWORD_FILE: /run/secrets/archivist_db_password
+        secrets: [archivist_db_password]
+        healthcheck:
+          test: ["CMD-SHELL", "pg_isready -U archivist -d archivist"]
+          interval: 10s
+          retries: 5
+
+      archivist-migrate:
+        image: simonsobs/archivist:v0.1.0
+        restart: "no"
+        entrypoint: ["alembic", "-c", "/archivist/alembic.ini", "upgrade", "head"]
+        volumes:
+          - /data/archivist/config:/etc/archivist
+        secrets: [archivist_db_password, archivist_authenticator]
+        depends_on:
+          archivist-database:
+            condition: service_healthy
+
+      archivist:
+        image: simonsobs/archivist:v0.1.0
+        restart: always
+        volumes:
+          - /data/archivist/config:/etc/archivist
+          - /gpfs/SIMONSOBS/so/tracked:/gpfs/SIMONSOBS/so/tracked:ro
+          - /tigerdata/so/archive:/tigerdata/so/archive
+        secrets: [archivist_db_password, archivist_authenticator]
+        depends_on:
+          archivist-migrate:
+            condition: service_completed_successfully
+
+    secrets:
+      archivist_db_password:
+        file: /data/archivist/secrets/db_password
+      archivist_authenticator:
+        file: /data/archivist/secrets/authenticator_password
+
+The migration container needs the secrets too, because loading the configuration
+reads every password file.
+
+Migrations
+----------
+
+To run the migrations by hand:
+
+.. code-block:: bash
+
+    podman compose run --rm archivist-migrate
+
+A database created before the archivist used migrations already has its tables but
+no migration record. Mark it as up to date once, before the first upgrade:
+
+.. code-block:: bash
+
+    podman compose run --rm --entrypoint alembic archivist-migrate \
+        -c /archivist/alembic.ini stamp head

docs/source/Librarians.rst

@@ -0,0 +1,83 @@
+Registering a Librarian
+=======================
+
+A librarian and an archivist talk in both directions: the librarian submits
+manifests, and the archivist calls back when an archive is stored. Both directions
+use HTTP Basic with the **same** ``user:password`` pair, so there is one credential
+to provision.
+
+Registering a librarian takes three steps: generate the credential, configure the
+archivist, and register the archivist on the librarian.
+
+Generate the Credential
+-----------------------
+
+Write a random password to a file readable by both services:
+
+.. code-block:: bash
+
+    python3 -c 'import secrets; print(secrets.token_urlsafe(32), end="")' > authenticator_password
+
+Avoid ``:`` in the password; the librarian splits the authenticator on it.
+
+Configure the Archivist
+-----------------------
+
+Add the librarian under both ``clients`` and ``librarians`` in the
+:doc:`configuration <Configuration>`, keyed by the librarian's own ``name``
+(from its ``server_settings.json``):
+
+.. code-block:: json
+
+    {
+      "clients": {
+        "princeton_librarian": {
+          "username": "archuser",
+          "password_file": "/run/secrets/archivist_authenticator"
+        }
+      },
+      "librarians": {
+        "princeton_librarian": {
+          "url": "http://librarian:21109",
+          "username": "archuser",
+          "password_file": "/run/secrets/archivist_authenticator"
+        }
+      }
+    }
+
+- ``clients`` lets the librarian submit manifests. The archivist rejects a manifest
+  whose ``librarian_name`` does not match the entry the credentials belong to.
+- ``librarians`` tells the archivist where to send callbacks. ``url`` is the
+  librarian's base URL; the callback path is added to it.
+- ``username`` is a free choice and is not the librarian's name.
+
+Restart the archivist to load the change.
+
+Register on the Librarian
+-------------------------
+
+On the librarian, register the archivist with the same credential:
+
+.. code-block:: bash
+
+    librarian add-archivist $LIBRARIAN_NAME --name=archivist-princeton
+                                            --url=http://archivist
+                                            --port=8080
+                                            --authenticator=archuser:$password
+
+Then add a ``create_archive`` background task that sends manifests to it. See the
+librarian documentation on
+`connecting an archivist <https://librarian.readthedocs.io/en/latest/Connections.html>`_
+and `background tasks <https://librarian.readthedocs.io/en/latest/Background.html>`_.
+
+Names That Must Match
+---------------------
+
+Most registration problems are a mismatch in one of these:
+
+- The librarian's ``name`` is the key under ``clients`` and ``librarians``.
+- The archivist's ``name`` is the ``--name`` given to ``add-archivist``, and the
+  ``archivist_name`` of the ``create_archive`` task.
+- The ``username`` and password are the two halves of ``--authenticator``.
+- The task's ``filesize_per_run`` is at most the archivist's ``maximal_size_bytes``.
+  A larger manifest is rejected, and the same files are selected again on every run.