Let's build a bootstrap gcc on Linux which targets an older glibc first.

qmk · Mar 6, 2024 · 4a08114 · 4a08114
1 parent c89e470
commit 4a08114
Show file tree

Hide file tree

Showing 19 changed files with 154 additions and 20 deletions.
diff --git a/.github/workflows/create_one_toolchain.yaml b/.github/workflows/create_one_toolchain.yaml
@@ -10,6 +10,8 @@ on:
       target_host: { type: string, required: true }
       build_script: { type: string, required: true }
       canadian_build: { type: boolean, required: false, default: false }
+      fetch_bootstrap: { type: boolean, required: false, default: true }
+      toolchain_suffix: { type: string, required: false, default: "" }
 
 jobs:
   # Split up the build as compiling newlib/libstdc++ targeting ARM takes so damn long.
@@ -23,6 +25,8 @@ jobs:
       target_host: ${{ inputs.target_host }}
       build_script: ${{ inputs.build_script }}
       canadian_build: ${{ inputs.canadian_build }}
+      fetch_bootstrap: ${{ inputs.fetch_bootstrap }}
+      toolchain_suffix: ${{ inputs.toolchain_suffix }}
       crosstool_args: 'build STOP=cc_for_build'
       restore_state: false
       phase_number: 1
@@ -38,6 +42,8 @@ jobs:
       target_host: ${{ inputs.target_host }}
       build_script: ${{ inputs.build_script }}
       canadian_build: ${{ inputs.canadian_build }}
+      fetch_bootstrap: ${{ inputs.fetch_bootstrap }}
+      toolchain_suffix: ${{ inputs.toolchain_suffix }}
       crosstool_args: 'build RESTART=cc_for_host STOP=libc_post_cc'
       phase_number: 2
 
@@ -52,6 +58,8 @@ jobs:
       target_host: ${{ inputs.target_host }}
       build_script: ${{ inputs.build_script }}
       canadian_build: ${{ inputs.canadian_build }}
+      fetch_bootstrap: ${{ inputs.fetch_bootstrap }}
+      toolchain_suffix: ${{ inputs.toolchain_suffix }}
       crosstool_args: 'build RESTART=companion_libs_for_target STOP=companion_libs_for_target'
       phase_number: 3
 
@@ -66,6 +74,8 @@ jobs:
       target_host: ${{ inputs.target_host }}
       build_script: ${{ inputs.build_script }}
       canadian_build: ${{ inputs.canadian_build }}
+      fetch_bootstrap: ${{ inputs.fetch_bootstrap }}
+      toolchain_suffix: ${{ inputs.toolchain_suffix }}
       crosstool_args: 'build RESTART=binutils_for_target'
       save_state: false
       toolchain_tarball: true
@@ -80,4 +90,4 @@ jobs:
       - name: Delete build state
         uses: geekyeggo/delete-artifact@v4
         with:
-            name: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}.tar
+            name: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}.tar
diff --git a/.github/workflows/create_toolchains.yaml b/.github/workflows/create_toolchains.yaml
@@ -67,7 +67,7 @@ jobs:
 
   tarballs:
     name: "Download required tarballs"
-    needs: crosstool
+    needs: [crosstool]
     runs-on: ubuntu-22.04
 
     steps:
@@ -100,9 +100,23 @@ jobs:
           name: tarballs.tar
           path: tarballs.tar
 
+  # Build a copy of linux/X64 toolchain to bootstrap the rest so that the resulting compilers are compatible with the old glibc.
+  bootstrap-linuxX64-toolchain:
+    name: "Build bootstrap linuxX64 toolchain"
+    needs: [tarballs]
+    uses: ./.github/workflows/create_one_toolchain.yaml
+    with:
+      runner_labels_json: '["ubuntu-22.04"]'
+      crosstool_host_os: linuxX64
+      build_host: linuxX64
+      target_host: linuxX64
+      build_script: x64linux-native-bootstrap.sh
+      fetch_bootstrap: false
+      toolchain_suffix: "_qmk_bootstrap"
+
   canadian-host-toolchains:
     name: "Build canadian host toolchains"
-    needs: [crosstool, tarballs]
+    needs: [bootstrap-linuxX64-toolchain]
     strategy:
       fail-fast: false
       matrix:
@@ -122,7 +136,7 @@ jobs:
 
   native-toolchains:
     name: "Build native toolchains"
-    needs: [crosstool, tarballs]
+    needs: [bootstrap-linuxX64-toolchain]
     strategy:
       fail-fast: false
       matrix:
@@ -151,7 +165,7 @@ jobs:
 
   canadian-toolchains:
     name: "Build canadian toolchains"
-    needs: [crosstool, tarballs, canadian-host-toolchains]
+    needs: [canadian-host-toolchains]
     strategy:
       fail-fast: false
       matrix:
@@ -179,7 +193,7 @@ jobs:
 
   native-linuxX64-toolchain:
     name: "Build native linuxX64 toolchain"
-    needs: [crosstool, tarballs]
+    needs: [bootstrap-linuxX64-toolchain]
     uses: ./.github/workflows/create_one_toolchain.yaml
     with:
       runner_labels_json: '["ubuntu-22.04"]'
@@ -190,7 +204,7 @@ jobs:
 
   canadian-windowsX64-toolchain:
     name: "Build native windowsX64 toolchain"
-    needs: [crosstool, tarballs, canadian-toolchains]
+    needs: [canadian-toolchains]
     uses: ./.github/workflows/create_one_toolchain.yaml
     with:
       runner_labels_json: '["ubuntu-22.04"]'
@@ -202,7 +216,7 @@ jobs:
 
   canadian-linuxARM64-toolchain:
     name: "Build native linuxARM64 toolchain"
-    needs: [crosstool, tarballs, canadian-toolchains]
+    needs: [canadian-toolchains]
     uses: ./.github/workflows/create_one_toolchain.yaml
     with:
       runner_labels_json: '["ubuntu-22.04"]'
@@ -214,7 +228,7 @@ jobs:
 
   publish:
     name: Publish toolchains
-    needs: [native-toolchains, canadian-toolchains, native-linuxX64-toolchain, canadian-windowsX64-toolchain, canadian-linuxARM64-toolchain]
+    needs: [bootstrap-linuxX64-toolchain, native-toolchains, canadian-toolchains, native-linuxX64-toolchain, canadian-windowsX64-toolchain, canadian-linuxARM64-toolchain]
     if: always() && !cancelled()
     runs-on: ubuntu-latest
 

diff --git a/.github/workflows/invoke_crosstool.yaml b/.github/workflows/invoke_crosstool.yaml
@@ -10,6 +10,8 @@ on:
       target_host: { type: string, required: true }
       build_script: { type: string, required: true }
       canadian_build: { type: boolean, required: false, default: false }
+      fetch_bootstrap: { type: boolean, required: false, default: true }
+      toolchain_suffix: { type: string, required: false, default: "" }
       phase_number: { type: number, required: true }
       crosstool_args: { type: string, required: false, default: "+finish" }
       restore_state: { type: boolean, required: false, default: true }
@@ -107,6 +109,19 @@ jobs:
         run: |
           tar xf tarballs.tar
 
+      - name: Download bootstrap toolchain
+        if: ${{ inputs.fetch_bootstrap }}
+        uses: actions/download-artifact@v4
+        with:
+          name: toolchain-host_${{ inputs.crosstool_host_os }}-target_${{ inputs.crosstool_host_os }}_qmk_bootstrap
+
+      - name: Extract bootstrap toolchain
+        if: ${{ inputs.fetch_bootstrap }}
+        run: |
+          mkdir -p toolchains
+          tar xf qmk_toolchain*host_${{ inputs.crosstool_host_os }}-target_${{ inputs.crosstool_host_os }}_qmk_bootstrap.tar.xz -C toolchains
+          rm qmk_toolchain*host_${{ inputs.crosstool_host_os }}-target_${{ inputs.crosstool_host_os }}_qmk_bootstrap.tar.xz
+
       - name: Download canadian host toolchain
         if: ${{ inputs.canadian_build }}
         uses: actions/download-artifact@v4
@@ -124,15 +139,15 @@ jobs:
         if: ${{ inputs.restore_state }}
         uses: actions/download-artifact@v4
         with:
-          name: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}
+          name: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}
 
       - name: Extract build state
         if: ${{ inputs.restore_state }}
         run: |
           # Ensure tarballs are extracted before we restore state as we've deleted $CT_COMMON_SRC_DIR before saving the state tarball
           ./${{ inputs.build_script }} +companion_tools_for_build
           # Extract the state tarball
-          tar xf state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}.tar
+          tar xf state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}.tar
 
       - name: Build toolchain
         run: |
@@ -151,7 +166,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: ${{ always() && !cancelled() }}
         with:
-          name: build.log-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}-phase${{ inputs.phase_number }}
+          name: build.log-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}-phase${{ inputs.phase_number }}
           path: |
             **/*.log
 
@@ -166,14 +181,14 @@ jobs:
             export STAT="stat"
           fi
           latest_state=$(dirname $(find $(find build -mindepth 4 -maxdepth 5 -type d -name state) -mindepth 2 -maxdepth 2 -type f -name env.sh -print0 | xargs -0r $STAT -c '%w|%n' | sort | cut -d'|' -f2 | tail -n1))
-          tar cf state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}.tar $latest_state
+          tar cf state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}.tar $latest_state
 
       - name: Upload build state
         if: ${{ inputs.save_state }}
         uses: actions/upload-artifact@v4
         with:
-          name: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}
-          path: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}.tar
+          name: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}
+          path: state-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}.tar
           overwrite: true
 
       - name: Make toolchain tarball
@@ -187,16 +202,16 @@ jobs:
           fi
           latest_state=$(dirname $(find $(find build -mindepth 4 -maxdepth 5 -type d -name state) -mindepth 2 -maxdepth 2 -type f -name env.sh -print0 | xargs -0r $STAT -c '%w|%n' | sort | cut -d'|' -f2 | tail -n1))
           source "$latest_state/env.sh"
-          tar cf qmk_toolchain-gcc${CT_GCC_VERSION}-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}.tar -C toolchains host_${{ inputs.build_host }}-target_${{ inputs.target_host }}
-          xz -T 0 -z -7 qmk_toolchain-gcc${CT_GCC_VERSION}-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}.tar
+          tar cf qmk_toolchain-gcc${CT_GCC_VERSION}-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}.tar -C toolchains host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}
+          xz -T 0 -z -7 qmk_toolchain-gcc${CT_GCC_VERSION}-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}.tar
 
       - name: Upload toolchain
         if: ${{ inputs.toolchain_tarball }}
         uses: actions/upload-artifact@v4
         with:
-          name: toolchain-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}
+          name: toolchain-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}
           path: |
-            qmk_toolchain-gcc*-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}.tar.xz
+            qmk_toolchain-gcc*-host_${{ inputs.build_host }}-target_${{ inputs.target_host }}${{ inputs.toolchain_suffix }}.tar.xz
 
       - name: Unmount case-sensitive workspace volume for macOS
         if: always()

diff --git a/aarch64linux-aarch64linux-canadian.sh b/aarch64linux-aarch64linux-canadian.sh
@@ -6,9 +6,14 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --sample-name=aarch64-rpi3-linux-gnu \
     --canadian-host=aarch64-unknown-linux-gnu \
     --vendor-name=unknown \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"
diff --git a/aarch64linux-native.sh b/aarch64linux-native.sh
@@ -6,8 +6,13 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --sample-name=aarch64-rpi3-linux-gnu \
     --vendor-name=unknown \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"
diff --git a/arm-aarch64linux-canadian.sh b/arm-aarch64linux-canadian.sh
@@ -6,6 +6,10 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --canadian-host=aarch64-unknown-linux-gnu \
     --sample-name=arm-none-eabi \
@@ -14,4 +18,5 @@ build_one \
     --binutils-plugins \
     --extra-newlib-nano \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"
diff --git a/arm-native.sh b/arm-native.sh
@@ -6,11 +6,16 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --sample-name=arm-none-eabi \
     --multilib-list=rmprofile \
     --libc=newlib \
     --binutils-plugins \
     --extra-newlib-nano \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"
diff --git a/arm-win64-canadian.sh b/arm-win64-canadian.sh
@@ -6,6 +6,10 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --canadian-host=x86_64-w64-mingw32 \
     --sample-name=arm-none-eabi \
@@ -14,4 +18,5 @@ build_one \
     --binutils-plugins \
     --extra-newlib-nano \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"
diff --git a/avr-aarch64linux-canadian.sh b/avr-aarch64linux-canadian.sh
@@ -6,9 +6,14 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --sample-name=avr \
     --canadian-host=aarch64-unknown-linux-gnu \
     --binutils-plugins \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"
diff --git a/avr-native.sh b/avr-native.sh
@@ -6,8 +6,13 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --sample-name=avr \
     --binutils-plugins \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"
diff --git a/avr-win64-canadian.sh b/avr-win64-canadian.sh
@@ -6,9 +6,14 @@ source "${script_dir}/common.bashinc"
 
 build_one_help "$@"
 
+if [[ $(uname -m) == "Linux" ]]; then
+    extra_args="--tools-prefix=x86_64-qmk_bootstrap-linux-gnu-"
+fi
+
 build_one \
     --sample-name=avr \
     --canadian-host=x86_64-w64-mingw32 \
     --binutils-plugins \
     --no-cross-gdb-python \
+    ${extra_args:-} \
     "$@"