use a wheel server as a cache for bootstrapping

by using a cache wheel server url, fromager can download already built wheels as and when needed to speed up the bootstrapping process by avoiding to actually build those wheels. Plus, if those wheels were built by fromager itself, then it can extract the build requirements files from it instead of recomputing them again.

Signed-off-by: Shubh Bapna <[email protected]>

.github/workflows/test.yaml

@@ -84,6 +84,7 @@ jobs:
           - bootstrap_build_tags
           - bootstrap_prerelease
           - bootstrap_constraints
+          - bootstrap_cache
           - build
           - build_order
           - build_steps

.mergify.yml

@@ -31,6 +31,7 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, bootstrap_build_tags, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build_order, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build_settings, ubuntu-latest)
@@ -56,6 +57,8 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_constraints, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_cache, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_cache, macos-latest)
           - check-success=e2e (3.12, 1.75, build, macos-latest)
           - check-success=e2e (3.12, 1.75, build, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, build_order, macos-latest)

e2e/test_bootstrap_build_tags.sh

@@ -20,6 +20,8 @@ EXPECTED_FILES="
 $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl
 "
 
+start_local_wheel_server
+
 pass=true
 for pattern in $EXPECTED_FILES; do
   if [ ! -f "${pattern}" ]; then
@@ -38,10 +40,10 @@ fromager \
   --wheels-repo="$OUTDIR/wheels-repo" \
   --work-dir="$OUTDIR/work-dir" \
   --settings-file="$SCRIPTDIR/bootstrap_settings.yaml" \
-  bootstrap 'stevedore==5.2.0'
+  bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
 
-if ! grep -q "stevedore: have wheel version 5.2.0: $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1" "$OUTDIR/bootstrap.log"; then
-  echo "FAIL: Did not find log message have wheel version in $OUTDIR/bootstrap.log" 1>&2
+if ! grep -q "stevedore: found built wheel on cache server" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: Did not find log message found built wheel on cache server in $OUTDIR/bootstrap.log" 1>&2
   pass=false
 fi
 
@@ -55,7 +57,7 @@ fromager \
   --wheels-repo="$OUTDIR/wheels-repo" \
   --work-dir="$OUTDIR/work-dir" \
   --settings-dir="$SCRIPTDIR/changelog_settings-2" \
-  bootstrap 'stevedore==5.2.0'
+  bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
 
 EXPECTED_FILES="
 $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-1*.whl

e2e/test_bootstrap_cache.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test bootstrap while taking advantage of the cache wheel server
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+pass=true
+
+# run fromager once to build wheels that can be used by a local wheel server
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  bootstrap 'stevedore==5.2.0'
+
+start_local_wheel_server
+rm -rf "$OUTDIR/sdists-repo"
+rm -rf "$OUTDIR/work-dir"
+
+# run fromager with the cache wheel server pointing to the local wheel server
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  bootstrap --cache-wheel-server-url=$WHEEL_SERVER_URL 'stevedore==5.2.0'
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+
+$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
+
+$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt
+
+$OUTDIR/work-dir/build-order.json
+$OUTDIR/work-dir/constraints.txt
+"
+
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+rm -rf "$OUTDIR/sdists-repo"
+rm -rf "$OUTDIR/work-dir"
+rm -rf "$OUTDIR/wheels-repo"
+
+# run fromager with the cache wheel server pointing to the pypi server
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/changelog_settings" \
+  bootstrap --cache-wheel-server-url="https://pypi.org/simple" 'stevedore==5.2.0'
+
+EXPECTED_FILES="
+$OUTDIR/wheels-repo/downloads/setuptools-*.whl
+$OUTDIR/wheels-repo/downloads/pbr-*.whl
+$OUTDIR/wheels-repo/downloads/stevedore-*.whl
+
+$OUTDIR/sdists-repo/downloads/stevedore-*.tar.gz
+$OUTDIR/sdists-repo/downloads/setuptools-*.tar.gz
+$OUTDIR/sdists-repo/downloads/pbr-*.tar.gz
+
+$OUTDIR/work-dir/build-order.json
+$OUTDIR/work-dir/constraints.txt
+"
+
+for pattern in $EXPECTED_FILES; do
+  if [ ! -f "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+NOT_EXPECTED_FILES="
+$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info/fromager-*.txt
+$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info/fromager-*.txt
+"
+
+for pattern in $NOT_EXPECTED_FILES; do
+  if [ -f "${pattern}" ]; then
+    echo "Found $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass
+
+EXPECTED_DIR="
+$OUTDIR/work-dir/pbr-*/pbr-*/pbr-*.dist-info
+$OUTDIR/work-dir/setuptools-*/setuptools-*/setuptools-*.dist-info
+$OUTDIR/work-dir/stevedore-*/stevedore-*/stevedore-*.dist-info
+"
+
+for pattern in $EXPECTED_DIR; do
+  if [ -d "${pattern}" ]; then
+    echo "Did not find $pattern" 1>&2
+    pass=false
+  fi
+done
+
+$pass