Upgrade continual learning benchmark

.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  lint-test-build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: Install package
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev,experiment,report]"
+      - name: Lint
+        run: ruff check .
+      - name: Format check
+        run: ruff format --check .
+      - name: Tests
+        run: pytest --cov
+      - name: CPU smoke benchmark
+        run: cl-bench run --config configs/smoke.yaml --method baseline --epochs 1 --device cpu --output-dir /tmp/cl-bench-runs
+      - name: CPU benchmark suite report
+        run: |
+          cl-bench suite \
+            --config configs/smoke.yaml \
+            --methods baseline replay derpp agem \
+            --seeds 7 \
+            --epochs 1 \
+            --device cpu \
+            --tracking json \
+            --output-dir /tmp/cl-bench-runs \
+            --report-dir /tmp/cl-bench-report
+      - name: Build package
+        run: python -m build
+      - name: Import check
+        run: python -c "import cl_bench; print(cl_bench.__version__)"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: benchmark-report-${{ matrix.python-version }}
+          path: /tmp/cl-bench-report

.gitignore

@@ -2,23 +2,39 @@
 __pycache__/
 *.py[cod]
 *$py.class
-*.pyc
 
-# Distribution / packaging
+# Build and packaging artifacts
 *.egg-info/
 dist/
 build/
+.ruff_cache/
+.pytest_cache/
+.coverage
+htmlcov/
 
 # Virtual environments
 .venv/
 venv/
 env/
 
-# Jupyter Notebook checkpoints
-.ipynb_checkpoints/
-
-# Results and logs (generated at runtime)
-results/logs/
+# Runtime artifacts generated by experiments
+data/
+checkpoints/
+logs/
+results/
+runs/
+wandb/
+tensorboard/
+mlruns/
+.hydra/
+*.pt
+*.pth
+*.ckpt
+*.npy
+*.npz
 
-# OS files
+# Notebook/editor/OS noise
+.ipynb_checkpoints/
 .DS_Store
+.idea/
+.vscode/

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY pyproject.toml README.md License ./
+COPY src ./src
+COPY configs ./configs
+COPY tests ./tests
+
+RUN python -m pip install --upgrade pip \
+    && python -m pip install -e ".[dev,experiment,report]"
+
+CMD ["cl-bench", "run", "--config", "configs/smoke.yaml", "--method", "baseline", "--epochs", "1", "--device", "cpu", "--output-dir", "/tmp/cl-bench-runs"]

Makefile

@@ -0,0 +1,39 @@
+PYTHON ?= python3
+VENV ?= .venv
+BIN := $(VENV)/bin
+export PYTHONPATH := src
+
+.PHONY: setup lint format test smoke suite benchmark build verify clean
+
+setup:
+	$(PYTHON) -m venv $(VENV)
+	$(BIN)/python -m pip install --upgrade pip
+	$(BIN)/python -m pip install -e ".[dev,experiment,report]"
+
+lint:
+	$(BIN)/ruff check .
+	$(BIN)/ruff format --check .
+
+format:
+	$(BIN)/ruff format .
+	$(BIN)/ruff check --fix .
+
+test:
+	$(BIN)/pytest --cov
+
+smoke:
+	$(BIN)/cl-bench run --config configs/smoke.yaml --method baseline --epochs 1 --device cpu
+
+suite:
+	$(BIN)/cl-bench suite --config configs/smoke.yaml --methods baseline ewc replay lwf derpp agem --seeds 7 --epochs 1 --device cpu --report-dir reports/smoke
+
+benchmark:
+	$(BIN)/cl-bench suite --config-name split_cifar10_headline --methods baseline ewc replay lwf derpp agem --seeds 13 21 --tracking both --report-dir docs/assets/split_cifar10_headline --title "Split CIFAR-10 Headline Benchmark"
+
+build:
+	$(BIN)/python -m build
+
+verify: lint test smoke build
+
+clean:
+	rm -rf build dist *.egg-info .pytest_cache .ruff_cache htmlcov .coverage