ci: refactor using uv by default

.github/workflows/benchmarks.yml

@@ -10,6 +10,8 @@ jobs:
     strategy:
       matrix:
         dataset: ["sift"]
+    env:
+      UV_PYTHON: "3.10"
     steps:
     - uses: actions/checkout@v4
       with:
@@ -19,21 +21,21 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.10"
-    - uses: ./.github/workflows/build_linux_wheel
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
       with:
-        python-minor-version: "10"
-    - name: Pip install
+        enable-cache: true
+    - name: Sync environment
       working-directory: python
       run: |
-        pip install $(ls target/wheels/*.whl)
-        pip install pandas
+        uv sync --frozen --extra tests --extra benchmarks
     - name: Run test
-      working-directory: benchmarks/${{ matrix.dataset }}
+      working-directory: python
       run: |
-        ./test_dataset.sh
+        uv run bash ../benchmarks/${{ matrix.dataset }}/test_dataset.sh
     - name: Archive results
       uses: actions/upload-artifact@v3
       with:
         name: ${{ matrix.dataset }}-results
         path: |
-          benchmarks/${{ matrix.dataset }}/benchmark.csv
+          benchmarks/${{ matrix.dataset }}/benchmark.csv

.github/workflows/ci-benchmarks.yml

@@ -10,10 +10,10 @@ jobs:
   bench_regress:
     timeout-minutes: 30
     runs-on: warp-custom-gcp-storage-benchmark
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
+env:
+  # Need up-to-date compilers for kernels
+  CC: clang-18
+  CXX: clang++-18
     defaults:
       run:
         shell: bash
@@ -33,28 +33,28 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.11 # Ray does not support 3.12 yet.
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build
+      - name: Sync environment
+        working-directory: python
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install maturin duckdb requests pytest pytest-benchmark
-          maturin develop --locked --release
+          uv sync --frozen --extra tests --extra benchmarks
       - name: Generate datasets
+        working-directory: python
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          python python/ci_benchmarks/datagen/gen_all.py
+          uv run python ci_benchmarks/datagen/gen_all.py
       - name: Run benchmarks
+        working-directory: python
         run: |
-          python -m venv venv
-          source venv/bin/activate
           bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} --adapter python_pytest \
-                      --branch main --testbed google-genoa --err --file results.json "python -mpytest --benchmark-json \
-                      results.json python/ci_benchmarks"
+                      --branch main --testbed google-genoa --err --file results.json "uv run python -mpytest --benchmark-json \
+                      results.json ci_benchmarks"

.github/workflows/file_verification.yml

@@ -9,6 +9,8 @@ jobs:
   run:
     timeout-minutes: 45
     runs-on: warp-ubuntu-latest-x64-8x
+    env:
+      UV_PYTHON: "3.11"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -18,6 +20,11 @@ jobs:
         with:
           python-version: "3.11"
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
       - name: Install Build Requirements
         run: |
           sudo apt update
@@ -49,16 +56,12 @@ jobs:
       - name: Build Lance
         working-directory: ./python
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install pyarrow maturin
-          maturin develop --release
+          uv sync --frozen --extra tests
 
       - name: Test Lance File Write Read Round Trip
+        working-directory: python
         run: |
-          source python/venv/bin/activate
-          cd .github/workflows/file_verification
-          python test_write_read.py
+          uv run python ../.github/workflows/file_verification/test_write_read.py
 
       - name: Cleanup
         run: |

.github/workflows/notebook.yml

@@ -19,6 +19,8 @@ jobs:
   linux:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
+    env:
+      UV_PYTHON: "3.10"
     steps:
     - uses: actions/checkout@v4
       with:
@@ -28,17 +30,16 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.10"
-    - uses: ./.github/workflows/build_linux_wheel
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
       with:
-        python-minor-version: "10"
-    - name: Pip install
+        enable-cache: true
+    - name: Sync environment
       working-directory: python
-      shell: bash
       run: |
-        pip3 install jupyter duckdb>=0.7
-        pip3 install $(ls target/wheels/*.whl)
+        uv sync --frozen --extra notebook
     - name: Run python tests
       shell: bash
       working-directory: notebooks
       run: |
-        jupyter nbconvert --to notebook --execute quickstart.ipynb
+        uv run --project ../python jupyter nbconvert --to notebook --execute quickstart.ipynb