🔨 engineering: refactor makefiles (#3303)

* 🔨 Refactor makefiles

Makefile

@@ -36,10 +36,10 @@ help:
 	@echo
 
 docs: .venv
-	poetry run mkdocs serve
+	.venv/bin/mkdocs serve
 
 watch-all:
-	poetry run watchmedo shell-command -c 'clear; make unittest; for lib in $(LIBS); do (cd $$lib && make unittest); done' --recursive --drop .
+	.venv/bin/watchmedo shell-command -c 'clear; make unittest; for lib in $(LIBS); do (cd $$lib && make unittest); done' --recursive --drop .
 
 test-all:
 	@echo '================ etl ================='
@@ -58,49 +58,29 @@ format-all:
 	done
 
 watch: .venv
-	poetry run watchmedo shell-command -c 'clear; make check-formatting lint check-typing coverage' --recursive --drop .
-
-.sanity-check:
-	@echo '==> Checking your Python setup'
-
-	@if python -c "import sys; exit(0 if sys.platform.startswith('win32') else 1)"; then \
-		echo 'ERROR: you are using a non-WSL Python interpreter, please consult the'; \
-		echo '       docs on how to swich to WSL Python on windows'; \
-		echo '       https://github.com/owid/etl/'; \
-		exit 1; \
-	fi
-	touch .sanity-check
+	.venv/bin/watchmedo shell-command -c 'clear; make check-formatting lint check-typing coverage' --recursive --drop .
 
 unittest: .venv
 	@echo '==> Running unit tests'
-	poetry run pytest -m "not integration" tests
+	.venv/bin/pytest -m "not integration" tests
 
 test: check-formatting check-linting check-typing unittest version-tracker
 
 test-integration: .venv
 	@echo '==> Running integration tests'
-	poetry run pytest -m integration tests
-
-.venv: .sanity-check pyproject.toml poetry.toml poetry.lock
-	@echo '==> Installing packages'
-	poetry install --no-ansi || poetry install --no-ansi
-	touch $@
-
-check-typing: .venv
-	@echo '==> Checking types'
-	poetry run pyright $(SRC)
+	.venv/bin/pytest -m integration tests
 
 coverage: .venv
 	@echo '==> Unit testing with coverage'
-	poetry run pytest --cov=etl --cov-report=term-missing tests
+	.venv/bin/pytest --cov=etl --cov-report=term-missing tests
 
 etl: .venv
 	@echo '==> Running etl on garden'
-	poetry run etl run garden
+	.venv/bin/etl run garden
 
 full: .venv
 	@echo '==> Running full etl'
-	poetry run etl run
+	.venv/bin/etl run
 
 clean:
 	@echo '==> Cleaning data/ folder'
@@ -113,19 +93,19 @@ clobber: clean
 
 lab: .venv
 	@echo '==> Starting Jupyter server'
-	poetry run jupyter lab
+	.venv/bin/jupyter lab
 
 publish: etl reindex
 	@echo '==> Publishing the catalog'
-	poetry run etl d publish --private
+	.venv/bin/etl d publish --private
 
 reindex: .venv
 	@echo '==> Creating a catalog index'
-	poetry run etl d reindex
+	.venv/bin/etl d reindex
 
 prune: .venv
 	@echo '==> Prune datasets with no recipe from catalog'
-	poetry run etl d prune
+	.venv/bin/etl d prune
 
 # Syncing catalog is useful if you want to avoid rebuilding it locally from scratch
 # which could take a few hours. This will download ~10gb from the main channels
@@ -137,12 +117,12 @@ sync.catalog: .venv
 
 grapher: .venv
 	@echo '==> Running full etl with grapher upsert'
-	poetry run etl run --grapher
+	.venv/bin/etl run --grapher
 
 dot: dependencies.pdf
 
 dependencies.pdf: .venv dag/main.yml etl/to_graphviz.py
-	poetry run etl graphviz dependencies.dot
+	.venv/bin/etl graphviz dependencies.dot
 	dot -Tpdf dependencies.dot >$@.tmp
 	mv -f $@.tmp $@
 
@@ -152,16 +132,16 @@ deploy:
 
 version-tracker: .venv
 	@echo '==> Check that no archive dataset is used by an active dataset, and that all active datasets are used'
-	poetry run etl d version-tracker
+	.venv/bin/etl d version-tracker
 
 api: .venv
 	@echo '==> Starting ETL API on http://localhost:8081/api/v1/indicators'
-	poetry run uvicorn api.main:app --reload --port 8081 --host 0.0.0.0
+	.venv/bin/uvicorn api.main:app --reload --port 8081 --host 0.0.0.0
 
 fasttrack: .venv
 	@echo '==> Starting Fast-track on http://localhost:8082/'
-	poetry run fasttrack --skip-auto-open --port 8082
+	.venv/bin/fasttrack --skip-auto-open --port 8082
 
 wizard: .venv
 	@echo '==> Starting Wizard on http://localhost:8053/'
-	poetry run etlwiz
+	.venv/bin/etlwiz

default.mk

@@ -17,8 +17,24 @@ help-default:
 # check formatting before lint, since an autoformat might fix linting issues
 test-default: check-formatting check-linting check-typing unittest
 
-.venv-default:
+.sanity-check:
+	@echo '==> Checking your Python setup'
+
+	@if python -c "import sys; exit(0 if sys.platform.startswith('win32') else 1)"; then \
+		echo 'ERROR: you are using a non-WSL Python interpreter, please consult the'; \
+		echo '       docs on how to swich to WSL Python on windows'; \
+		echo '       https://github.com/owid/etl/'; \
+		exit 1; \
+	fi
+	touch .sanity-check
+
+
+.venv-default: .sanity-check
 	@echo '==> Installing packages'
+	@if [ -n "$(PYTHON_VERSION)" ]; then \
+		echo '==> Using Python version $(PYTHON_VERSION)'; \
+		poetry env use python$(PYTHON_VERSION); \
+	fi
 	poetry install
 	touch .venv
 
@@ -36,30 +52,40 @@ check-default:
 
 lint-default: .venv
 	@echo '==> Linting & Sorting imports'
-	@poetry run ruff check --fix $(SRC)
+	@.venv/bin/ruff check --fix $(SRC)
 
 check-linting-default: .venv
 	@echo '==> Checking linting'
-	@poetry run ruff check $(SRC)
+	@.venv/bin/ruff check $(SRC)
 
 check-formatting-default: .venv
 	@echo '==> Checking formatting'
-	@poetry run ruff format --check $(SRC)
+	@.venv/bin/ruff format --check $(SRC)
 
 check-typing-default: .venv
 	@echo '==> Checking types'
-	poetry run pyright $(SRC)
+	. .venv/bin/activate && .venv/bin/pyright $(SRC)
 
 unittest-default: .venv
 	@echo '==> Running unit tests'
-	poetry run pytest $(SRC)
+	.venv/bin/pytest $(SRC)
 
 format-default: .venv
 	@echo '==> Reformatting files'
-	@poetry run ruff format $(SRC)
+	@.venv/bin/ruff format $(SRC)
+
+coverage-default: .venv
+	@echo '==> Unit testing with coverage'
+	.venv/bin/pytest --cov=owid --cov-report=term-missing tests
 
 watch-default: .venv
-	poetry run watchmedo shell-command -c 'clear; make test' --recursive --drop .
+	@echo '==> Watching for changes and re-running checks'
+	.venv/bin/watchmedo shell-command -c 'clear; make check' --recursive --drop .
+
+bump-default: .venv
+	@echo '==> Bumping version'
+	.venv/bin/bump2version --no-tag  --no-commit $(filter-out $@, $(MAKECMDGOALS))
+
 
 # allow you to override a command, e.g. "watch", but if you do not, then use
 # the default

docs/getting-started/building-datasets.md

@@ -38,7 +38,7 @@ OK (0.0s)
 
     ```bash
     # Install the dependencies defined in pyproject.toml and poetry.lock
-    poetry install
+    make .venv
     ```
 
     Next you can use one of the following two methods:

lib/catalog/.github/workflows/python-package.yml

@@ -30,7 +30,7 @@ jobs:
 
     - name: Install packages
       run: |
-        poetry install
+        make .venv
 
     - name: Check formatting
       run: |

lib/catalog/Makefile

@@ -6,23 +6,3 @@
 include ../../default.mk
 
 SRC = owid tests
-
-# watch:
-# 	poetry run watchmedo shell-command -c 'clear; make unittest' --recursive --drop .
-
-.venv: poetry.toml pyproject.toml poetry.lock
-	@echo '==> Installing packages'
-	poetry install
-	touch .venv
-
-check-typing: .venv
-	@echo '==> Checking types'
-	poetry run pyright $(SRC)
-
-coverage: .venv
-	@echo '==> Unit testing with coverage'
-	poetry run pytest --cov=owid --cov-report=term-missing tests
-
-watch: .venv
-	@echo '==> Watching for changes and re-running tests'
-	poetry run watchmedo shell-command -c 'clear; make check-formatting lint check-typing coverage' --recursive --drop .

lib/datautils/.github/workflows/python-package.yml

@@ -30,7 +30,7 @@ jobs:
 
     - name: Install packages
       run: |
-        poetry install
+        make .venv
 
     - name: Check formatting
       run: |

lib/datautils/Makefile

@@ -6,34 +6,3 @@
 include ../../default.mk
 
 SRC = owid tests
-
-# watch:
-# 	poetry run watchmedo shell-command -c 'clear; make unittest' --recursive --drop .
-
-
-report: report-coverage report-linting report-server-launch
-	@poetry run python -m http.server .reports/
-
-check-typing: .venv
-	@echo '==> Checking types'
-	poetry run pyright owid tests
-
-report-coverage: .venv
-	@echo '==> Unit testing with coverage'
-	poetry run pytest --cov=owid --cov-report=term-missing --cov-report=html:.reports/coverage --cov-report=xml tests
-
-report-linting: .venv
-	@echo '==> Linting'
-	@poetry run flake8 --format=html --htmldir=.reports/linting owid || true
-
-report-server-launch: .venv
-	@echo '==> Showing reports'
-	@poetry run python -m http.server --directory .reports/
-
-watch: .venv
-	@echo '==> Watching for changes and re-running tests'
-	poetry run watchmedo shell-command -c 'clear; make check-formatting lint check-typing coverage' --recursive --drop .
-
-bump: .venv
-	@echo '==> Bumping version'
-	poetry run bump2version --no-tag  --no-commit $(filter-out $@, $(MAKECMDGOALS))

lib/datautils/README.md

@@ -1,7 +1,7 @@
 # owid-datautils
 
 ![version](https://img.shields.io/badge/version-0.5.3-blue)
-![version](https://img.shields.io/badge/python-3.8|3.9|3.10-blue.svg?&logo=python&logoColor=yellow) 
+![version](https://img.shields.io/badge/python-3.8|3.9|3.10-blue.svg?&logo=python&logoColor=yellow)
 [![Build status](https://badge.buildkite.com/caba621fb64f2c7dcc692a474e68f4ead21e6ba6ee151fe3b6.svg)](https://buildkite.com/our-world-in-data/owid-datautils-unit-tests)
 [![Documentation Status](https://readthedocs.org/projects/owid-datautils/badge/?version=latest)](https://docs.owid.io/projects/owid-datautils/en/latest/?badge=latest)
 
@@ -29,7 +29,7 @@ pip install poetry
 ### Install in development mode
 
 ```
-poetry install
+make .venv
 ```
 
 ### Test the code

lib/repack/.github/workflows/python-package.yml

@@ -30,7 +30,7 @@ jobs:
 
     - name: Install packages
       run: |
-        poetry install
+        make .venv
 
     - name: Check formatting
       run: |

lib/repack/Makefile

@@ -6,15 +6,3 @@
 include ../../default.mk
 
 SRC = owid tests
-
-check-typing: .venv
-	@echo '==> Checking types'
-	poetry run pyright $(SRC)
-
-coverage: .venv
-	@echo '==> Unit testing with coverage'
-	poetry run pytest --cov=owid --cov-report=term-missing tests
-
-watch: .venv
-	@echo '==> Watching for changes and re-running tests'
-	poetry run watchmedo shell-command -c 'clear; make check-formatting lint check-typing coverage' --recursive --drop .

lib/walden/Makefile

@@ -12,17 +12,13 @@ default:
 	@echo '  make clean     Delete any fetched data files'
 	@echo
 
-.venv: pyproject.toml poetry.toml poetry.lock
-	poetry install || poetry install
-	test -d .venv && touch $@
-
 audit: .venv
 	@echo '==> Auditing JSON records'
-	poetry run python owid/walden/audit.py
+	.venv/bin/python owid/walden/audit.py
 
 fetch: .venv
 	@echo '==> Fetching the full dataset'
-	@poetry run python owid/walden/fetch.py
+	@.venv/bin/python owid/walden/fetch.py
 
 clean:
 	@echo '==> Deleting all downloaded data'
@@ -32,33 +28,30 @@ test: check-formatting lint check-typing unittest audit
 
 lint: .venv
 	@echo '==> Linting'
-	@poetry run flake8 owid
+	@.venv/bin/flake8 owid
 
 check-formatting: .venv
 	@echo '==> Checking formatting'
-	@poetry run black --check owid/walden
-	@poetry run black --check ingests/
-	@poetry run python -m owid.walden.format_json --check
+	@.venv/bin/black --check owid/walden
+	@.venv/bin/black --check ingests/
+	@.venv/bin/python -m owid.walden.format_json --check
 	@echo '==> Checking imports sorting'
-	@poetry run isort --check-only owid/walden/
-	@poetry run isort --check-only ingests/
+	@.venv/bin/isort --check-only owid/walden/
+	@.venv/bin/isort --check-only ingests/
 
 check-typing: .venv
 	@echo '==> Checking types'
-	@poetry run pyright owid/walden
+	. .venv/bin/activate && .venv/bin/pyright owid/walden
 
 unittest: .venv
 	@echo '==> Running unit tests'
-	@PYTHONPATH=. poetry run pytest
+	@PYTHONPATH=. .venv/bin/pytest
 
 format: .venv
 	@echo '==> Reformatting files'
-	@poetry run black -q owid/walden/
-	@poetry run black -q ingests/
-	@poetry run python owid/walden/format_json.py
+	@.venv/bin/black -q owid/walden/
+	@.venv/bin/black -q ingests/
+	@.venv/bin/python owid/walden/format_json.py
 	@echo '==> Sorting imports'
-	@poetry run isort -q owid/walden/
-	@poetry run isort -q ingests/
-
-watch: .venv
-	poetry run watchmedo shell-command -c 'clear; make test' --recursive --drop .
+	@.venv/bin/isort -q owid/walden/
+	@.venv/bin/isort -q ingests/