Skip to content

Commit

Permalink
Add cross compilation for Python (#24)
Browse files Browse the repository at this point in the history
  • Loading branch information
virtuald authored Oct 18, 2023
1 parent 2a8b0a0 commit eb618a4
Show file tree
Hide file tree
Showing 21 changed files with 1,050 additions and 9 deletions.
75 changes: 75 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ name: CI

on: [push, pull_request, workflow_dispatch]

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
build:
name: Build
Expand All @@ -14,3 +18,74 @@ jobs:
docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
make push
if: ${{ github.repository_owner == 'wpilibsuite' && github.ref == 'refs/heads/main' }}
build-minimal:
name: Build Minimal
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- run: make update
- run: make build/minimal-cross
- run: |
docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
make push/minimal-cross
if: ${{ github.repository_owner == 'wpilibsuite' && github.ref == 'refs/heads/main' }}
- run: make save/minimal-cross
- uses: actions/upload-artifact@v3
with:
name: roborio
path: roborio.tar.gz
retention-days: 1
- uses: actions/upload-artifact@v3
with:
name: raspbian
path: raspbian.tar.gz
retention-days: 1
- uses: actions/upload-artifact@v3
with:
name: aarch64
path: aarch64.tar.gz
retention-days: 1

build-python:
name: Build Python Images
needs: [build-minimal]
runs-on: ubuntu-latest
strategy:
matrix:
type:
- raspbian
- aarch64
pyversion:
- py38
- py39
- py310
- py311
- py312
include:
- type: roborio
pyversion: py312

steps:
- uses: actions/checkout@v3

- name: Download image
uses: actions/download-artifact@v3
with:
name: ${{ matrix.type }}
path: img

- name: Load image
run: |
gunzip -c img/${{ matrix.type }}.tar.gz | docker load
rm img/${{ matrix.type }}.tar.gz
- name: Build image
run: make build/cross-${{ matrix.type }}-${{ matrix.pyversion }}

- name: Upload image
run: |
docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
make push/cross-${{ matrix.type }}-${{ matrix.pyversion }}
if: ${{ github.repository_owner == 'wpilibsuite' && github.ref == 'refs/heads/main' }}
41 changes: 35 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ OPENSDK_UBUNTU?=20.04
DOCKER_USER?=wpilib

.PHONY: usage update push build \
$(patsubst %,build/%, base cross opensdl)
$(patsubst %,push/%, base cross opensdl)
$(patsubst %,build/%, base cross minimal-base minimal-cross opensdk)
$(patsubst %,push/%, base cross minimal-base minimal-cross opensdk)
usage:
@echo "Run make update, make build, and make push"

Expand All @@ -23,12 +23,24 @@ build/base:

build/cross: build/base
cd roborio-cross-ubuntu && \
docker build -t ${DOCKER_USER}/roborio-cross-ubuntu:2024-${UBUNTU} -f Dockerfile.2024 --build-arg UBUNTU=${UBUNTU} .
docker build -t ${DOCKER_USER}/roborio-cross-ubuntu:2024-${UBUNTU} -f Dockerfile.2024 --build-arg UBUNTU=${UBUNTU} --build-arg TYPE=base .
cd raspbian-cross-ubuntu && \
docker build -t ${DOCKER_USER}/raspbian-cross-ubuntu:bullseye-${UBUNTU} -f Dockerfile.bullseye --build-arg UBUNTU=${UBUNTU} .
docker build -t ${DOCKER_USER}/raspbian-cross-ubuntu:bullseye-${UBUNTU} -f Dockerfile.bullseye --build-arg UBUNTU=${UBUNTU} --build-arg TYPE=base .
cd aarch64-cross-ubuntu && \
docker build -t ${DOCKER_USER}/aarch64-cross-ubuntu:bullseye-${UBUNTU} -f Dockerfile.bullseye --build-arg UBUNTU=${UBUNTU} .

docker build -t ${DOCKER_USER}/aarch64-cross-ubuntu:bullseye-${UBUNTU} -f Dockerfile.bullseye --build-arg UBUNTU=${UBUNTU} --build-arg TYPE=base .

build/minimal-base:
cd ubuntu-minimal-base && \
docker build -t ${DOCKER_USER}/ubuntu-minimal-base:${UBUNTU} -f Dockerfile.${UBUNTU} .

build/minimal-cross: build/minimal-base
cd roborio-cross-ubuntu && \
docker build -t ${DOCKER_USER}/roborio-cross-ubuntu-minimal:2024-${UBUNTU} -f Dockerfile.2024 --build-arg UBUNTU=${UBUNTU} --build-arg TYPE=minimal-base .
cd raspbian-cross-ubuntu && \
docker build -t ${DOCKER_USER}/raspbian-cross-ubuntu-minimal:bullseye-${UBUNTU} -f Dockerfile.bullseye --build-arg UBUNTU=${UBUNTU} --build-arg TYPE=minimal-base .
cd aarch64-cross-ubuntu && \
docker build -t ${DOCKER_USER}/aarch64-cross-ubuntu-minimal:bullseye-${UBUNTU} -f Dockerfile.bullseye --build-arg UBUNTU=${UBUNTU} --build-arg TYPE=minimal-base .

build/opensdk:
cd opensdk/ubuntu && \
docker build -t ${DOCKER_USER}/opensdk-ubuntu:${OPENSDK_UBUNTU} --build-arg OPENSDK_UBUNTU=${OPENSDK_UBUNTU} .
Expand All @@ -44,5 +56,22 @@ push/cross: push/base
docker push ${DOCKER_USER}/raspbian-cross-ubuntu:bullseye-${UBUNTU}
docker push ${DOCKER_USER}/aarch64-cross-ubuntu:bullseye-${UBUNTU}

push/minimal-base:
docker push ${DOCKER_USER}/ubuntu-minimal-base:${UBUNTU}

push/minimal-cross: push/minimal-base
docker push ${DOCKER_USER}/roborio-cross-ubuntu-minimal:2024-${UBUNTU}
docker push ${DOCKER_USER}/raspbian-cross-ubuntu-minimal:bullseye-${UBUNTU}
docker push ${DOCKER_USER}/aarch64-cross-ubuntu-minimal:bullseye-${UBUNTU}

push/opensdk:
docker push ${DOCKER_USER}/opensdk-ubuntu:${OPENSDK_UBUNTU}


.PHONY: save/minimal-cross
save/minimal-cross:
docker save ${DOCKER_USER}/roborio-cross-ubuntu-minimal:2024-${UBUNTU} | gzip > roborio.tar.gz
docker save ${DOCKER_USER}/raspbian-cross-ubuntu-minimal:bullseye-${UBUNTU} | gzip > raspbian.tar.gz
docker save ${DOCKER_USER}/aarch64-cross-ubuntu-minimal:bullseye-${UBUNTU} | gzip > aarch64.tar.gz

include cross-ubuntu-py/py.mk
15 changes: 15 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,40 @@ ubuntu-base
- Base image for cross-compiler images
- Published as wpilib/ubuntu-base:host ubuntu version (e.g. 18.04)

ubuntu-minimal-base
- Based on official ubuntu repo (host)
- Base image for minimal cross-compiler images that only contain cross compiler
- Published as wpilib/ubuntu-minimal-base:host ubuntu version (e.g. 18.04)

roborio-cross-ubuntu
- Based on wpilib/ubuntu-base
- Cross-compiler for roboRIO
- Published as wpilib/roborio-cross-ubuntu:frc season-host ubuntu version
(e.g. 2020-18.04)
- Minimal variant published as roborio-cross-ubuntu-minimal

raspbian-cross-ubuntu
- Based on wpilib/ubuntu-base
- Cross-compiler for Raspbian
- Published as wpilib/raspbian-cross-ubuntu:target version-host ubuntu version
(e.g. 10-18.04)
- Minimal variant published as raspbian-cross-ubuntu-minimal

aarch64-cross-ubuntu
- Based on wpilib/ubuntu-base
- Cross-compiler for aarch64 (TX2 and Nano)
- Published as wpilib/aarch-cross-ubuntu:target version-host ubuntu version
(e.g. bionic-18.04)
- Minimal variant published as raspbian-cross-ubuntu-aarch64

gazebo-ubuntu
- Based on wpilib/ubuntu-base
- Installs gazebo and protobuf required to build gazebo plugins
- Published as wpilib/gazebo-ubuntu:host ubuntu version (e.g. 18.04)

roborio-cross-ubuntu-pyXXX
raspbian-cross-ubuntu-pyXXX
aarch64-cross-ubuntu-pyXXX
- Based on the minimal cross compiler image
- Contains crossenv at /build/venv for cross-compiling python wheels
- Published as wpilib/{type}-cross-ubuntu:{version}-{ubuntu}-py{pyversion}
3 changes: 2 additions & 1 deletion aarch64-cross-ubuntu/Dockerfile.bullseye
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
ARG UBUNTU=22.04
FROM wpilib/ubuntu-base:${UBUNTU}
ARG TYPE=base
FROM wpilib/ubuntu-${TYPE}:${UBUNTU}

# Install toolchain
RUN curl -SL https://github.com/wpilibsuite/opensdk/releases/download/v2024-1/arm64-bullseye-2024-x86_64-linux-gnu-Toolchain-10.2.0.tgz | sh -c 'mkdir -p /usr/local && cd /usr/local && tar xzf - --strip-components=2'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
From 1325a6365032f2c7dea4f823003f46305724a6a6 Mon Sep 17 00:00:00 2001
From: Dustin Spicuzza <[email protected]>
Date: Sat, 1 Jan 2022 22:44:03 -0500
Subject: [PATCH] Use specified $host_cpu when cross compiling for Linux

Previously arm processors were hardcoded to "arm", which doesn't match
the output of uname -m (used for native compilation)
---
configure | 8 +-------
configure.ac | 8 +-------
2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/configure b/configure
index 2e7e0b7e0a..d91928fb79 100755
--- a/configure
+++ b/configure
@@ -3383,13 +3383,7 @@ $as_echo "\"$MACHDEP\"" >&6; }
if test "$cross_compiling" = yes; then
case "$host" in
*-*-linux*)
- case "$host_cpu" in
- arm*)
- _host_cpu=arm
- ;;
- *)
- _host_cpu=$host_cpu
- esac
+ _host_cpu=$host_cpu
;;
*-*-cygwin*)
_host_cpu=
diff --git a/configure.ac b/configure.ac
index 0c06914147..897c3a00ef 100644
--- a/configure.ac
+++ b/configure.ac
@@ -438,13 +438,7 @@ AC_SUBST(_PYTHON_HOST_PLATFORM)
if test "$cross_compiling" = yes; then
case "$host" in
*-*-linux*)
- case "$host_cpu" in
- arm*)
- _host_cpu=arm
- ;;
- *)
- _host_cpu=$host_cpu
- esac
+ _host_cpu=$host_cpu
;;
*-*-cygwin*)
_host_cpu=
--
2.31.1

Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
From 0bca228707c3f69cb8e6b4bc59b2ecf54f1f7c76 Mon Sep 17 00:00:00 2001
From: Dustin Spicuzza <[email protected]>
Date: Sat, 3 Oct 2020 12:31:44 -0400
Subject: [PATCH] bpo-41916: allow cross-compiled python to have -pthread set
for CXX

When cross-compiling, the compile/run test for -pthread always fails so -pthread
will never be automatically set without an override from the cache. ac_cv_pthread
can already be overridden, so do the same thing for ac_cv_cxx_thread.
---
configure | 12 +++++++++---
configure.ac | 8 +++++---
2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/configure b/configure
index 0914e24704..119d720500 100755
--- a/configure
+++ b/configure
@@ -7710,12 +7710,14 @@ fi

# If we have set a CC compiler flag for thread support then
# check if it works for CXX, too.
-ac_cv_cxx_thread=no
if test ! -z "$CXX"
then
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CXX also accepts flags for thread support" >&5
$as_echo_n "checking whether $CXX also accepts flags for thread support... " >&6; }
-ac_save_cxx="$CXX"
+if ${ac_cv_cxx_thread+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_save_cxx="$CXX"

if test "$ac_cv_kpthread" = "yes"
then
@@ -7744,10 +7746,14 @@ then
fi
rm -fr conftest*
fi
+CXX="$ac_save_cxx"
+fi
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_thread" >&5
$as_echo "$ac_cv_cxx_thread" >&6; }
+else
+ ac_cv_cxx_thread=no
fi
-CXX="$ac_save_cxx"


# checks for header files
diff --git a/configure.ac b/configure.ac
index 7051dc109a..6259c0290c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2080,11 +2080,11 @@ fi

# If we have set a CC compiler flag for thread support then
# check if it works for CXX, too.
-ac_cv_cxx_thread=no
if test ! -z "$CXX"
then
AC_MSG_CHECKING(whether $CXX also accepts flags for thread support)
-ac_save_cxx="$CXX"
+AC_CACHE_VAL(ac_cv_cxx_thread,
+[ac_save_cxx="$CXX"

if test "$ac_cv_kpthread" = "yes"
then
@@ -2113,9 +2113,11 @@ then
fi
rm -fr conftest*
fi
+CXX="$ac_save_cxx"])
AC_MSG_RESULT($ac_cv_cxx_thread)
+else
+ ac_cv_cxx_thread=no
fi
-CXX="$ac_save_cxx"

dnl # check for ANSI or K&R ("traditional") preprocessor
dnl AC_MSG_CHECKING(for C preprocessor type)
--
2.26.2

Loading

0 comments on commit eb618a4

Please sign in to comment.