🔨 Switch from poetry to uv package manager

.github/workflows/publish-owid-catalog.yml

@@ -19,12 +19,14 @@ jobs:
       with:
         python-version: '3.x'
 
-    - name: Install Poetry
-      run: curl -sSL https://install.python-poetry.org | python3 -
+    - name: Install UV
+      run: curl -LsSf https://astral.sh/uv/install.sh | sh
 
     - name: Publish
       env:
-        POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }}
+        TWINE_USERNAME: ourworldindata
+        TWINE_PASSWORD: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }}
       run: |
         cd lib/catalog &&
-        poetry publish --build -u ourworldindata
+        uv build &&
+        uvx twine upload dist/*

.readthedocs.yaml

@@ -7,9 +7,9 @@ build:
   commands:
     - git submodule init
     - git submodule update
-    - pip install -U poetry
-    - poetry install
-    - poetry run mkdocs build
+    - pip install -U uv
+    - uv install
+    - uv run mkdocs build
     - mkdir -p _readthedocs
     - cp -r site/ _readthedocs/html/
 

Makefile

@@ -2,7 +2,7 @@
 #  Makefile
 #
 
-.PHONY: etl docs full lab test-default publish grapher dot watch clean clobber deploy api
+.PHONY: etl docs full lab test-default publish grapher dot watch clean clobber deploy api activate
 
 include default.mk
 

apps/owidbot/data_diff.py

@@ -72,7 +72,7 @@ def format_etl_diff(lines: list[str]) -> Tuple[str, str]:
 
 def call_etl_diff(include: str) -> list[str]:
     cmd = [
-        "poetry",
+        "uv",
         "run",
         "etl",
         "diff",

apps/utils/profile.py

@@ -14,7 +14,7 @@
 
 To profile grapher upserts, it is better to use cProfile and run something like this:
 ```
-ssh owid@staging-site-my-branch "cd etl && poetry run python -m cProfile -s cumtime etl/command.py grapher://grapher/biodiversity/2024-01-25/cherry_blossom --grapher --only --force --workers 1" | head -n 100
+ssh owid@staging-site-my-branch "cd etl && uv run python -m cProfile -s cumtime etl/command.py grapher://grapher/biodiversity/2024-01-25/cherry_blossom --grapher --only --force --workers 1" | head -n 100
 ```
 """
 

apps/wizard/app_pages/metaplay.py

@@ -58,7 +58,7 @@ def run_steps() -> None:
     """
     # env_path = paths.BASE_DIR / ".env.X"
     # subprocess.run(f"export $(cat {env_path} | xargs)", shell=True)
-    # subprocess.run(["poetry", "run", "etl", DUMMY, "--grapher"], env=dict(**os.environ, DEBUG="1"))
+    # subprocess.run(["uv", "run", "etl", DUMMY, "--grapher"], env=dict(**os.environ, DEBUG="1"))
     # setting DEBUG to true runs much faster
     config.DEBUG = True
     etl_main([DUMMY], grapher=True, workers=1)

apps/wizard/etl_steps/express.py

@@ -509,7 +509,7 @@ def export_metadata() -> None:
             ## Run step
             st.markdown("##### Run ETL meadow step")
             st.code(
-                f"poetry run etl run {form.meadow_step_uri} {'--private' if form.is_private else ''}",
+                f"uv run etl run {form.meadow_step_uri} {'--private' if form.is_private else ''}",
                 language="shellSession",
             )
 
@@ -519,7 +519,7 @@ def export_metadata() -> None:
             st.markdown("##### Harmonize country names")
             st.markdown("Run it in your terminal:")
             st.code(
-                f"poetry run etl harmonize data/meadow/{form.base_step_name}/{form.short_name}.feather country etl/steps/data/garden/{form.base_step_name}.countries.json",
+                f"uv run etl harmonize data/meadow/{form.base_step_name}/{form.short_name}.feather country etl/steps/data/garden/{form.base_step_name}.countries.json",
                 "shellSession",
             )
             st.markdown("Or run it on Wizard")
@@ -533,7 +533,7 @@ def export_metadata() -> None:
             st.markdown("##### Run ETL step")
             st.markdown("After editing the code of your Garden step, run the following command:")
             st.code(
-                f"poetry run etl run {form.garden_step_uri} {'--private' if form.is_private else ''}",
+                f"uv run etl run {form.garden_step_uri} {'--private' if form.is_private else ''}",
                 "shellSession",
             )
 

apps/wizard/etl_steps/garden.py

@@ -426,7 +426,7 @@ def export_metadata() -> None:
             st.markdown("####  1. Harmonize Country names")
             st.markdown("Run it in your terminal:")
             st.code(
-                f"poetry run etl harmonize data/meadow/{form.namespace}/{form.meadow_version}/{form.short_name}/{form.short_name}.feather country etl/steps/data/garden/{form.namespace}/{form.version}/{form.short_name}.countries.json",
+                f"uv run etl harmonize data/meadow/{form.namespace}/{form.meadow_version}/{form.short_name}/{form.short_name}.feather country etl/steps/data/garden/{form.namespace}/{form.version}/{form.short_name}.countries.json",
                 "shellSession",
             )
             st.markdown("Or run it on Wizard")
@@ -440,7 +440,7 @@ def export_metadata() -> None:
             st.markdown("####  2. Run ETL step")
             st.markdown("After editing the code of your Garden step, run the following command:")
             st.code(
-                f"poetry run etl run data{private_suffix}://garden/{form.namespace}/{form.version}/{form.short_name} {'--private' if form.is_private else ''}",
+                f"uv run etl run data{private_suffix}://garden/{form.namespace}/{form.version}/{form.short_name} {'--private' if form.is_private else ''}",
                 "shellSession",
             )
             # 3/ Optional stuff
@@ -466,13 +466,13 @@ def export_metadata() -> None:
                 with st.container(border=True):
                     st.markdown("Alternitavely you can generate the metadata with the following command:")
                     st.code(
-                        f"poetry run etl metadata-export {st.session_state['garden.dataset_path']}",
+                        f"uv run etl metadata-export {st.session_state['garden.dataset_path']}",
                         "shellSession",
                     )
 
                 st.markdown("then manual edit it and rerun the step again with")
                 st.code(
-                    f"poetry run etl run data{private_suffix}://garden/{form.namespace}/{form.version}/{form.short_name} {'--private' if form.is_private else ''}",
+                    f"uv run etl run data{private_suffix}://garden/{form.namespace}/{form.version}/{form.short_name} {'--private' if form.is_private else ''}",
                     "shellSession",
                 )
 

apps/wizard/etl_steps/grapher.py

@@ -244,7 +244,7 @@ def update_state() -> None:
 
             Then run the grapher step:
             ```
-            poetry run etl run grapher/{form.namespace}/{form.version}/{form.short_name} --grapher {"--private" if form.is_private else ""}
+            uv run etl run grapher/{form.namespace}/{form.version}/{form.short_name} --grapher {"--private" if form.is_private else ""}
             ```
             Your new dataset should then appear in your local set-up: http://localhost:3030/admin/datasets. Follow the instructions [here](https://github.com/owid/owid-grapher/blob/master/docs/docker-compose-mysql.md) to create your local Grapher development set-up.
 
@@ -264,7 +264,7 @@ def update_state() -> None:
             After you run
 
             ```
-            ENV=.env.staging poetry run etl run grapher/{form.namespace}/{form.version}/{form.short_name} --grapher {"--private" if form.is_private else ""}
+            ENV=.env.staging uv run etl run grapher/{form.namespace}/{form.version}/{form.short_name} --grapher {"--private" if form.is_private else ""}
             ```
 
             you should see it [in staging admin](https://staging.owid.cloud/admin/datasets).
@@ -283,7 +283,7 @@ def update_state() -> None:
             and running
 
             ```
-            ENV=.env.prod poetry run etl run grapher/{form.namespace}/{form.version}/{form.short_name} --grapher {"--private" if form.is_private else ""}
+            ENV=.env.prod uv run etl run grapher/{form.namespace}/{form.version}/{form.short_name} --grapher {"--private" if form.is_private else ""}
             ```
 
         4. Check your dataset in [admin](https://owid.cloud/admin/datasets).

apps/wizard/etl_steps/meadow.py

@@ -282,7 +282,7 @@ def update_state() -> None:
             # 1/ Run ETL step
             st.markdown("#### 1. Run ETL step")
             st.code(
-                f"poetry run etl run data{private_suffix}://meadow/{form.namespace}/{form.version}/{form.short_name} {'--private' if form.is_private else ''}",
+                f"uv run etl run data{private_suffix}://meadow/{form.namespace}/{form.version}/{form.short_name} {'--private' if form.is_private else ''}",
                 language="shellSession",
             )
 

apps/wizard/etl_steps/snapshot.py

@@ -841,8 +841,8 @@ def run_snap_step() -> None:
     script_path = f"{SNAPSHOTS_DIR}/{form.namespace}/{form.snapshot_version}/{form.short_name}.py"
 
     # Build command
-    command = f"poetry run python {script_path}"
-    commands = ["poetry", "run", "python", script_path]
+    command = f"uv run python {script_path}"
+    commands = ["uv", "run", "python", script_path]
     if form.dataset_manual_import:
         # Get snapshot local file
         commands.extend(["--path-to-file", st.session_state["snapshot_file"]])

default.mk

@@ -28,15 +28,20 @@ test-default: check-formatting check-linting check-typing unittest
 	fi
 	touch .sanity-check
 
+install-uv-default:
+	@if ! command -v uv >/dev/null 2>&1; then \
+		echo '==> UV not found. Installing...'; \
+		echo $$HOME; \
+		curl -LsSf https://astral.sh/uv/install.sh | sh && . $$HOME/.cargo/env; \
+	fi
 
-.venv: .sanity-check
+.venv-default: install-uv .sanity-check
 	@echo '==> Installing packages'
 	@if [ -n "$(PYTHON_VERSION)" ]; then \
 		echo '==> Using Python version $(PYTHON_VERSION)'; \
-		poetry env use python$(PYTHON_VERSION); \
+		export UV_PYTHON=$(PYTHON_VERSION); \
 	fi
-	poetry install
-	touch .venv
+	$(HOME)/.cargo/bin/uv sync --all-extras
 
 check-default:
 	@echo '==> Lint & Format & Typecheck changed files'

docs/architecture/metadata/structuring-yaml.md

@@ -26,10 +26,10 @@ tables:
 To generate a metadata YAML file with pre-populated variable names for an existing garden dataset, execute:
 
 ```
-poetry run etl metadata export data/garden/my_namespace/my_version/my_dataset
+uv run etl metadata export data/garden/my_namespace/my_version/my_dataset
 ```
 
-Check `poetry run etl metadata-export --help` for more options.
+Check `uv run etl metadata-export --help` for more options.
 
 ## Handling Multi-line Strings and Whitespace
 

docs/contributing.md

@@ -16,7 +16,7 @@ All our data work is public and open-source. Our ETL work is available on GitHub
 To build any of our datasets, you first need [to install the ETL](../getting-started/working-environment/) and then execute any step you want to build:
 
 ```
-poetry run etl run <step>
+uv run etl run <step>
 ```
 
 ## Report issues

docs/data/faostat.md

@@ -209,10 +209,10 @@ Using data from garden, we create an additional dataset in the `explorers` chann
 These are the steps OWID follows to ensure that FAOSTAT data is up-to-date, or to update one or more datasets for
 which there is new data (let us call the new dataset version to be created `YYYY-MM-DD`):
 
-0. Activate the etl poetry environment (from the root folder of the etl repository):
+0. Activate the etl virtual environment (from the root folder of the etl repository):
 
 ```bash
-  poetry shell
+  . .venv/bin/activate
 ```
 
 1.  Execute the ingestion script, to fetch data for any dataset that may have been updated in FAOSTAT.

docs/dev/docs.md

@@ -18,7 +18,7 @@ The markdown files powering the documentation are in the same repository, under
 First, make sure that you have all the development libraries installed
 
 ```bash
-poetry install --with dev
+make .venv
 ```
 
 ### Preview the documentation

docs/getting-started/building-datasets.md

@@ -34,26 +34,26 @@ OK (0.0s)
 
     If you are not already, make sure to use these tools within your **virtual environment**.
 
-    Details can be found in the [Poetry docs](https://python-poetry.org/docs/basic-usage/#using-your-virtual-environment), but if you have never used the tool before here is a basic usage that will get you started (this assumes you have been [here](../working-environment/#install-dependencies) and installed Poetry):
+    Details can be found in the [UV docs](https://docs.astral.sh/uv/getting-started/), but if you have never used the tool before here is a basic usage that will get you started (this assumes you have been [here](../working-environment/#install-dependencies) and installed UV):
 
     ```bash
-    # Install the dependencies defined in pyproject.toml and poetry.lock
+    # Install the dependencies defined in pyproject.toml and uv.lock
     make .venv
     ```
 
     Next you can use one of the following two methods:
 
     ```bash
     # Activate virtual environment. Once activated, you can use the tools here as normal.
-    poetry shell
+    make activate
     etl run --dry-run data://garden/biodiversity/2024-01-25/cherry_blossom
     ```
 
     Or
 
     ```bash
     # Run a specific command in virtual environment. This will not leave you in the virtual environment.
-    poetry run etl run --dry-run data://garden/biodiversity/2024-01-25/cherry_blossom
+    uv run etl run --dry-run data://garden/biodiversity/2024-01-25/cherry_blossom
     ```
 
 

docs/getting-started/working-environment.md

@@ -24,10 +24,10 @@ You will need Python 3.10+, basic build tools, and MySQL client libraries.
     xcode-select --install
     ```
 
-    Then install Python 3.9+, MySQL client and [poetry](https://python-poetry.org/). Poetry is our preferred python packaging and dependency management tool.
+    Then install Python 3.9+ and MySQL client and [UV](https://docs.astral.sh/uv/). UV is our preferred python packaging and dependency management tool.
 
     ```bash
-    brew install python mysql-client poetry pkg-config
+    brew install python mysql-client uv pkg-config
     ```
 
     You then need to inform Python where to find MySQL by adding some lines to your `~/.zshrc` file (or `~/.bash_profile`, depends on your shell). Run `brew info mysql-client` to see what's needed. For example, on an M1/M2 Mac where Homebrew installs to `/opt/homebrew`, you would need to add:
@@ -56,14 +56,19 @@ You will need Python 3.10+, basic build tools, and MySQL client libraries.
     sudo apt install python3-dev python3-virtualenv python3-setuptools mysql-client libmysqlclient-dev
     ```
 
-    However, the version of Poetry that ships with Ubuntu is too old, so we need to install a more recent version.
+    Then install UV package manager with
 
-    The [recommended method](https://python-poetry.org/docs/#installation) is to run:
+    ```bash
+    curl -LsSf https://astral.sh/uv/install.sh | sh
+    ```
+
+    or
 
     ```bash
-    curl -sSL https://install.python-poetry.org | python3 -
+    pip install uv
     ```
 
+
 === "Windows"
 
     You will need to [install WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) to get started.
@@ -103,6 +108,10 @@ You will need Python 3.10+, basic build tools, and MySQL client libraries.
 
 ## Install pyenv
 
+!!! tip
+
+    `pyenv` is not crucial now after switching to `uv` as a package manager. However, it is still recommended to use it to manage your Python versions.
+
 Even though it's not compulsory, it is **highly recommended** to install [pyenv](https://github.com/pyenv/pyenv#installation) to manage your Python versions. This will allow you to have multiple Python versions installed in your machine and switch between them easily. You will also avoid issues caused by updating system wide Python.
 
 Follow the instructions in the [pyenv installation guide](https://github.com/pyenv/pyenv#installation) or follow the steps below.

docs/guides/adding-data.md

@@ -138,7 +138,7 @@ git checkout -b data/new-dataset
 
 2. **Add snapshot data**
     ```bash
-    poetry run python snapshots/<namespace>/<version>/<dataset_short_name>.py
+    uv run python snapshots/<namespace>/<version>/<dataset_short_name>.py
     ```
 
 ### Meadow step

docs/guides/data-work/add-data.md

@@ -157,7 +157,7 @@ git checkout -b data/new-dataset
 
 2. **Add snapshot data**
     ```bash
-    poetry run python snapshots/<namespace>/<version>/<dataset_short_name>.py
+    uv run python snapshots/<namespace>/<version>/<dataset_short_name>.py
     ```
 
 ### Meadow step

docs/guides/harmonize-countries.md

@@ -74,7 +74,7 @@ If a mapping file already exists, it will resume where the mapping file left off
 As an example, start the harmonization interactive session for table `undp_hdr` from dataset `meadow/un/2024-04-09/undp_hdr`, which has `country` column with the raw country names:
 
 ```bash
-poetry run etl harmonize data/meadow/un/2024-04-09/undp_hdr/undp_hdr.feather country mapping.json
+uv run etl harmonize data/meadow/un/2024-04-09/undp_hdr/undp_hdr.feather country mapping.json
 206 countries/regions to harmonize
   └ 188 automatically matched
   └ 18 ambiguous countries/regions

etl/scripts/faostat/create_new_snapshots.py

@@ -10,15 +10,15 @@
 Usage:
 * To show available options:
 ```
-poetry run python -m create_new_snapshots -h
+uv run python -m create_new_snapshots -h
 ```
 * To simply check if any of the datasets needs to be updated (without actually creating snapshots):
 ```
-poetry run python -m create_new_snapshots -r
+uv run python -m create_new_snapshots -r
 ```
 * To check for updates and actually create new snapshots:
 ```
-poetry run python -m create_new_snapshots
+uv run python -m create_new_snapshots
 ```
 
 """

etl/steps/__init__.py

@@ -598,7 +598,7 @@ def _run_py(self) -> None:
         if sys.platform == "linux":
             args.extend(["prlimit", f"--as={config.MAX_VIRTUAL_MEMORY_LINUX}"])
 
-        args.extend(["poetry", "run", "etl", "d", "run-python-step"])
+        args.extend(["uv", "run", "etl", "d", "run-python-step"])
 
         if config.IPDB_ENABLED:
             args.append("--ipdb")
@@ -610,8 +610,12 @@ def _run_py(self) -> None:
             ]
         )
 
+        # Add uv to the path, it causes problems in Buildkite
+        env = os.environ.copy()
+        env["PATH"] = os.path.expanduser("~/.cargo/bin") + ":" + env["PATH"]
+
         try:
-            subprocess.check_call(args, env=os.environ.copy())
+            subprocess.check_call(args, env=env)
         except subprocess.CalledProcessError:
             # swallow this exception and just exit -- the important stack trace
             # will already have been printed to stderr