Migrate Python library to uv (#345)

It's fast and seems to be the direction Python projects are going. We
also now spin up a server for each test, for better isolation. This
required some changes to the piping on the server side, to allow us to
use port `0` to get the OS to assign an available port. This means we
need to compute the connection string _after_ we have bound the port, so
that the connection string contains the actual bound port and not port
`0`.

This also sets up Python tests to run in CI/CD, but only when the Python
code changes (at least for now).

.github/workflows/python-tests.yml

@@ -0,0 +1,34 @@
+name: Run Python tests
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - "python/**"
+      - ".github/workflows/python-tests.yml"
+
+jobs:
+  uv-example:
+    name: python
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./python
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run tests
+        run: uv run pytest tests
+
+      - name: Check formatting
+        run: uv run ruff format --check

crates/y-sweet/src/main.rs

@@ -8,6 +8,7 @@ use std::{
     path::PathBuf,
 };
 use tokio::io::AsyncReadExt;
+use tokio::net::TcpListener;
 use tokio_util::sync::CancellationToken;
 use tracing::metadata::LevelFilter;
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
@@ -180,6 +181,9 @@ async fn main() -> Result<()> {
                 *port,
             );
 
+            let listener = TcpListener::bind(addr).await?;
+            let addr = listener.local_addr()?;
+
             let store = if let Some(store) = store {
                 let store = get_store_from_opts(store)?;
                 store.init().await?;
@@ -207,7 +211,7 @@ async fn main() -> Result<()> {
 
             let prod = *prod;
             let handle = tokio::spawn(async move {
-                server.serve(&addr, prod).await.unwrap();
+                server.serve(listener, prod).await.unwrap();
             });
 
             tracing::info!("Listening on ws://{}", addr);
@@ -312,8 +316,11 @@ async fn main() -> Result<()> {
                 *port,
             );
 
+            let listener = TcpListener::bind(addr).await?;
+            let addr = listener.local_addr()?;
+
             tokio::spawn(async move {
-                server.serve_doc(&addr, false).await.unwrap();
+                server.serve_doc(listener, false).await.unwrap();
             });
 
             tracing::info!("Listening on http://{}", addr);

crates/y-sweet/src/server.rs

@@ -360,11 +360,10 @@ impl Server {
 
     async fn serve_internal(
         self: Arc<Self>,
-        addr: &SocketAddr,
+        listener: TcpListener,
         redact_errors: bool,
         routes: Router,
     ) -> Result<()> {
-        let listener = TcpListener::bind(addr).await?;
         let token = self.cancellation_token.clone();
 
         let app = if redact_errors {
@@ -383,16 +382,16 @@ impl Server {
         Ok(())
     }
 
-    pub async fn serve(self, addr: &SocketAddr, redact_errors: bool) -> Result<()> {
+    pub async fn serve(self, listener: TcpListener, redact_errors: bool) -> Result<()> {
         let s = Arc::new(self);
         let routes = s.routes();
-        s.serve_internal(addr, redact_errors, routes).await
+        s.serve_internal(listener, redact_errors, routes).await
     }
 
-    pub async fn serve_doc(self, addr: &SocketAddr, redact_errors: bool) -> Result<()> {
+    pub async fn serve_doc(self, listener: TcpListener, redact_errors: bool) -> Result<()> {
         let s = Arc::new(self);
         let routes = s.single_doc_routes();
-        s.serve_internal(addr, redact_errors, routes).await
+        s.serve_internal(listener, redact_errors, routes).await
     }
 
     fn verify_doc_token(&self, token: Option<&str>, doc: &str) -> Result<(), AppError> {

python/.gitignore

@@ -19,3 +19,4 @@ var/
 .pytest_cache/
 .venv/
 .ruff_cache/
+test-out

python/.python-version

@@ -0,0 +1 @@
+3.12

python/README.md

@@ -36,28 +36,41 @@ async with (
 Use a Yjs client like [ypy-websocket](https://davidbrochart.github.io/ypy-websocket/usage/client/) or [pycrdt](https://github.com/jupyter-server/pycrdt)
 in conjunction with `y_sweet_sdk` to access the actual Y.Doc data.
 
-## Installation
+## Developing
 
-For development installation with test dependencies:
+Developing `y_sweet_sdk` requires the [`uv`](https://docs.astral.sh/uv/) project manager.
+
+To install it on Mac or Liunux, run:
 
 ```bash
-pip install -e ".[dev]"
+curl -LsSf https://astral.sh/uv/install.sh | sh
 ```
 
-## Tests
+(See [the docs](https://docs.astral.sh/uv/) for other platforms and more information.)
+
+When using `uv`, you do not need to manage a virtual environment yourself. Instead, you interact with
+Python using the `uv` command, which automatically picks up the virtual environment from the location.
 
-First run a y-sweet server:
+To set up the virtual environment for development, run:
 
 ```bash
-npx y-sweet serve
+uv sync --dev
 ```
 
-Then run the tests:
+This installs both the regular dependencies and the development dependencies.
+
+### Tests
+
+Once commands are installed in your virtual environment, you can run them with `uv run`.
+
+To run tests, run:
 
 ```bash
-pytest
+uv run pytest
 ```
 
-## Development
+This runs the `pytest` command in the virtual environment.
+
+### Formatting
 
-Run `ruff format` to format before committing changes.
+Run `uv run ruff format` to format before committing changes.

python/pyproject.toml

@@ -18,21 +18,21 @@ dependencies = [
     "requests~=2.32.2",
     "pycrdt~=0.9.11",
 ]
+requires-python = ">=3.12"
 
 [project.urls]
 Homepage = "https://github.com/jamsocket/y-sweet"
 
-[project.optional-dependencies]
-dev = [
-    "build==1.2.2",
-    "pytest~=8.3.3",
-    "twine~=5.1.1",
-    "ruff~=0.6.5",
-]
-
 [tool.setuptools]
 package-dir = { "" = "src" }
 
 [tool.pytest.ini_options]
 addopts = "-ra -q"
 testpaths = ["tests"]
+pythonpath = [".", "src"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.3.3",
+    "ruff>=0.8.0",
+]

python/tests/server.py

@@ -0,0 +1,55 @@
+import subprocess
+import signal
+from typing import Optional
+from os import path, environ
+import json
+import time
+import re
+
+
+BASE_DIR = path.join(path.dirname(__file__), "..", "..", "crates", "y-sweet")
+
+
+class Server:
+    @staticmethod
+    def build():
+        subprocess.run(
+            ["cargo", "build"],
+            cwd=BASE_DIR,
+            check=True,
+        )
+
+    def __init__(self, test_id: str):
+        self.data_dir = path.join(path.dirname(__file__), "..", "test-out", test_id)
+        self.process = subprocess.Popen(
+            ["cargo", "run", "--", "serve", self.data_dir, "--port", "0"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=BASE_DIR,
+            env={"Y_SWEET_LOG_JSON": "true", **environ},
+            text=True,
+            bufsize=1,
+        )
+
+        # Wait for connection string in the output
+        while self.process.poll() is None:
+            line = self.process.stdout.readline()
+            print("here", line)
+            match = re.search(r"CONNECTION_STRING=([^ ]+)", line)
+            if match:
+                self.connection_string = match.group(1)
+                break
+
+        if not self.connection_string:
+            raise RuntimeError("Server failed to start and provide connection string")
+
+    def shutdown(self):
+        if self.process:
+            self.process.terminate()
+            try:
+                self.process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self.process.kill()
+
+            # Clean up the process
+            self.process = None

python/tests/test_y_sweet.py

@@ -6,44 +6,49 @@
 import random
 import string
 import pycrdt as Y
-
-CONNECTION_STRING = environ.get("CONNECTION_STRING", "ys://localhost:8080")
+from .server import Server
 
 
 class TestYSweet(unittest.TestCase):
+    def setUpClass():
+        Server.build()
+
     def setUp(self):
-        self.random_string = "".join(
-            random.choices(string.ascii_lowercase + string.digits, k=10)
-        )
+        self.test_id = self.id().split(".")[-1]
+        self.server = Server(self.test_id)
+        self.connection_string = self.server.connection_string
+
+    def tearDown(self):
+        self.server.shutdown()
 
     def test_create_doc(self):
-        doc = DocumentManager(CONNECTION_STRING)
-        name = f"{self.random_string}-test-doc"
+        doc = DocumentManager(self.connection_string)
+        name = "test-doc"
         result = doc.create_doc(name)
         self.assertEqual(result["docId"], name)
 
     def test_get_client_token(self):
-        doc = DocumentManager(CONNECTION_STRING)
+        doc = DocumentManager(self.connection_string)
 
         # getting a non-existent token should raise an error
-        nonexistent = f"{self.random_string}-nonexistent"
+        nonexistent = "nonexistent"
         with self.assertRaises(YSweetError):
             doc.get_client_token(nonexistent)
 
-        existing = f"{self.random_string}-existing"
+        existing = "existing"
         doc.create_doc(existing)
         result = doc.get_client_token(existing)
         self.assertEqual(result["docId"], existing)
 
     def test_get_url(self):
-        doc = DocumentManager(CONNECTION_STRING)
-        name = f"{self.random_string}-test-doc"
+        doc = DocumentManager(self.connection_string)
+        name = "test-doc"
 
         doc.get_websocket_url(name)
 
     def test_get_update(self):
-        dm = DocumentManager(CONNECTION_STRING)
-        name = f"{self.random_string}-test-doc"
+        dm = DocumentManager(self.connection_string)
+        name = "test-doc"
         dm.create_doc(name)
         conn = dm.get_connection(name)
 
@@ -65,8 +70,8 @@ def test_get_update(self):
         self.assertEqual(text2.to_py(), "Hello, world!")
 
     def test_get_update_direct(self):
-        dm = DocumentManager(CONNECTION_STRING)
-        name = f"{self.random_string}-test-doc"
+        dm = DocumentManager(self.connection_string)
+        name = "test-doc"
         dm.create_doc(name)
 
         # Generate an update to apply
@@ -87,8 +92,8 @@ def test_get_update_direct(self):
         self.assertEqual(text2.to_py(), "Hello, world!")
 
     def test_update_context(self):
-        dm = DocumentManager(CONNECTION_STRING)
-        name = f"{self.random_string}-test-doc"
+        dm = DocumentManager(self.connection_string)
+        name = "test-doc"
         dm.create_doc(name)
         conn = dm.get_connection(name)
 
@@ -102,5 +107,6 @@ def test_update_context(self):
         text2 = doc2.get("text", type=Y.Text)
         self.assertEqual(text2.to_py(), "Hello, world!")
 
+
 if __name__ == "__main__":
     unittest.main()