Skip to content

Commit

Permalink
Backport support for building cross compilers
Browse files Browse the repository at this point in the history
This is the squashed combination of the following commits:

- Add the missing `$(EXE)` for `stripdebug` invocations
- Use `target` instead of `host` to detect the C toolchain
- Use `target` instead of `host` when relevant in configuration
- Detect a _build_ C toolchain to build `sak`
- Check that the OCaml versions are compatible for a cross compiler
- Use a `TARGET_BINDIR` configure variable instead of `--with-target-bindir`
- Add a configurable library directory on target
- Detect `flexlink` only on relevant targets
- Add a `Config` entry for the target OS type
- Add a Makefile.cross with rules to build a cross compiler
- Enable bootstrapping flexdll in the cross-compiler setting
- Add cross-compilation cases to `sak`
- Add a CI workflow to test cross compilers
- Fix static builds of the compiler
  • Loading branch information
shym committed Jan 8, 2025
1 parent 1ccb919 commit 1515ed9
Show file tree
Hide file tree
Showing 19 changed files with 2,705 additions and 378 deletions.
4 changes: 4 additions & 0 deletions .depend
Original file line number Diff line number Diff line change
Expand Up @@ -4951,6 +4951,7 @@ middle_end/closure/closure.cmo : \
typing/env.cmi \
lambda/debuginfo.cmi \
middle_end/convert_primitives.cmi \
utils/config.cmi \
middle_end/compilenv.cmi \
utils/clflags.cmi \
middle_end/clambda_primitives.cmi \
Expand All @@ -4973,6 +4974,7 @@ middle_end/closure/closure.cmx : \
typing/env.cmx \
lambda/debuginfo.cmx \
middle_end/convert_primitives.cmx \
utils/config.cmx \
middle_end/compilenv.cmx \
utils/clflags.cmx \
middle_end/clambda_primitives.cmx \
Expand Down Expand Up @@ -5162,6 +5164,7 @@ middle_end/flambda/closure_conversion.cmo : \
middle_end/flambda/flambda.cmi \
lambda/debuginfo.cmi \
middle_end/convert_primitives.cmi \
utils/config.cmi \
middle_end/compilation_unit.cmi \
middle_end/flambda/base_types/closure_origin.cmi \
middle_end/flambda/base_types/closure_id.cmi \
Expand Down Expand Up @@ -5190,6 +5193,7 @@ middle_end/flambda/closure_conversion.cmx : \
middle_end/flambda/flambda.cmx \
lambda/debuginfo.cmx \
middle_end/convert_primitives.cmx \
utils/config.cmx \
middle_end/compilation_unit.cmx \
middle_end/flambda/base_types/closure_origin.cmx \
middle_end/flambda/base_types/closure_id.cmx \
Expand Down
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ META.in typo.missing-header

# Github templates and scripts lack headers, have long lines
/.github/** typo.missing-header typo.long-line=may typo.very-long-line=may
/.github/workflows/build-cross.yml typo.non-ascii

/.mailmap typo.long-line typo.missing-header typo.non-ascii
/CONTRIBUTING.md typo.non-ascii=may
Expand Down
269 changes: 269 additions & 0 deletions .github/workflows/build-cross.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,269 @@
name: Cross compilers

on:
pull_request:
types: [opened, synchronize, reopened, labeled, unlabeled]
workflow_dispatch:

# Restrict the GITHUB_TOKEN
permissions: {}

# See build.yml
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' || github.sha }}
cancel-in-progress: true

env:
res: 0
TESTDIR: >-
C:\Бактріан🐫
STR_UTF8: >-
"C:\\Бактріан🐫"
STR_UTF16: >-
L"C:\\\x0411\x0430\x043a\x0442\x0440\x0456\x0430\x043d\xd83d\xdc2b"
EXAMPLE_PROGRAM: |
let _ =
Printf.printf "Version: %s\nOS: %s\nUnix: %b\nWin: %b\nCygwin: %b\n"
Sys.ocaml_version Sys.os_type Sys.unix Sys.win32 Sys.cygwin
COMPLIBS_PROG_X86_64: |
let _ =
Printf.printf "allow_unaligned_access = %b\n" Arch.allow_unaligned_access;
Printf.printf "win64 = %b\n" Arch.win64
COMPLIBS_PROG_AARCH64: |
let _ =
Printf.printf "allow_unaligned_access = %b\n" Arch.allow_unaligned_access;
Printf.printf "macosx = %b\n" Arch.macosx
jobs:
non-cross:
if: contains(github.event.pull_request.labels.*.name, 'run-crosscompiler-tests')
runs-on: ubuntu-latest
steps:
- name: Checkout OCaml
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Configure, build and install OCaml
run: |
PREFIX="$HOME/.local"
echo "$PREFIX/bin" >> "$GITHUB_PATH"
set -x
./configure --disable-warn-error --disable-ocamldoc \
--disable-ocamltest --disable-stdlib-manpages \
--disable-dependency-generation --prefix="$PREFIX" || res=$?
if ! [ "$res" = 0 ]; then cat config.log; exit "$res"; fi
make -j
make install
cd "$HOME"
tar caf /tmp/ocaml.tar.zst .local
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: non-cross-ocaml
path: /tmp/ocaml.tar.zst
retention-days: 1

cross-windows:
runs-on: ubuntu-latest
needs: non-cross
steps:
- name: Download Artifact
uses: actions/download-artifact@v4
with:
name: non-cross-ocaml
- name: Install non-cross OCaml and set up environment
run: |
set -x
tar xaf ocaml.tar.zst -C "$HOME"
rm -f ocaml.tar.zst
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
sudo apt-get install -y gcc-mingw-w64-x86-64
- name: Checkout OCaml
uses: actions/checkout@v4
with:
submodules: true
persist-credentials: false
- name: Configure, build and install Linux-to-Windows OCaml
run: |
set -x
./configure --prefix="$HOME/cross" --target=x86_64-w64-mingw32 \
TARGET_LIBDIR="$TESTDIR" || res=$?
if ! [ "$res" = 0 ]; then cat config.log; exit "$res"; fi
make crossopt -j$(nproc)
make installcross
ln -sr "$HOME/cross/bin/flexlink.opt.exe" "$HOME/.local/bin/flexlink"
- name: Show opt.opt configuration
run: |
set -x
$HOME/cross/bin/ocamlopt.opt.exe -config
cat runtime/build_config.h
- name: Cross compile a small program
run: |
printf %s "$EXAMPLE_PROGRAM$COMPLIBS_PROG_X86_64" > example.ml
set -x
cat example.ml
$HOME/cross/bin/ocamlopt.opt.exe -I $HOME/cross/lib/ocaml/compiler-libs/ ocamlcommon.cmxa ocamloptcomp.cmxa example.ml -o example.exe -verbose
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: windows-executable
path: example.exe
retention-days: 1
- name: Test cross sak
run: |
printf %s "$STR_UTF16" > utf16.ref
printf %s "$STR_UTF8" > utf8.ref
set -x
runtime/sak.exe encode-C-utf16-literal "$TESTDIR" > utf16
git diff --no-index utf16.ref utf16
runtime/sak.exe encode-C-utf8-literal "$TESTDIR" > utf8
git diff --no-index utf8.ref utf8
run-windows:
runs-on: windows-latest
needs: cross-windows
steps:
- name: Download Artifact
uses: actions/download-artifact@v4
with:
name: windows-executable
- name: Run example program
run: |
.\example.exe
cross-arm-linux:
runs-on: ubuntu-latest
needs: non-cross
steps:
- name: Download Artifact
uses: actions/download-artifact@v4
with:
name: non-cross-ocaml
- name: Install non-cross OCaml and set up environment
run: |
set -x
tar xaf ocaml.tar.zst -C "$HOME"
rm -f ocaml.tar.zst
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
sudo apt-get install -y gcc-aarch64-linux-gnu qemu-user
- name: Checkout OCaml
uses: actions/checkout@v4
with:
submodules: true
persist-credentials: false
- name: Configure, build and install Linux-to-Windows OCaml
run: |
set -x
./configure --prefix="$HOME/cross" --target=aarch64-linux-gnu \
|| res=$?
if ! [ "$res" = 0 ]; then cat config.log; exit "$res"; fi
make crossopt -j
make installcross
- name: Show opt.opt configuration
run: |
set -x
$HOME/cross/bin/ocamlopt.opt -config
cat runtime/build_config.h
- name: Cross compile a small program
run: |
printf %s "$EXAMPLE_PROGRAM$COMPLIBS_PROG_AARCH64" > example.ml
set -x
cat example.ml
$HOME/cross/bin/ocamlopt.opt -I $HOME/cross/lib/ocaml/compiler-libs/ ocamlcommon.cmxa ocamloptcomp.cmxa example.ml -o example -verbose
- name: Run the small example program
run: |
set -x
qemu-aarch64 -L /usr/aarch64-linux-gnu example
- name: Test cross sak
run: |
printf %s "$STR_UTF16" > utf16.ref
printf %s "$STR_UTF8" > utf8.ref
set -x
runtime/sak encode-C-utf16-literal "$TESTDIR" > utf16
git diff --no-index utf16.ref utf16
runtime/sak encode-C-utf8-literal "$TESTDIR" > utf8
git diff --no-index utf8.ref utf8
cross-android:
runs-on: ubuntu-latest
needs: non-cross
steps:
- name: Download Artifact
uses: actions/download-artifact@v4
with:
name: non-cross-ocaml
- name: Install non-cross OCaml
run: |
set -x
tar xaf ocaml.tar.zst -C "$HOME"
rm -f ocaml.tar.zst
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Restore the Android NDK from cache
uses: actions/cache/restore@v4
id: cache
with:
path: |
/home/runner/android
key: android-ndk
- name: Download the Android NDK
run: |
set -x
mkdir -p "$HOME/android"
cd "$HOME/android"
wget --no-verbose https://dl.google.com/android/repository/android-ndk-r27b-linux.zip
unzip android-ndk-r27b-linux.zip
rm android-ndk-r27b-linux.zip
if: steps.cache.outputs.cache-hit != 'true'
- name: Save the Android NDK to cache
uses: actions/cache/save@v4
with:
path: |
/home/runner/android
key: android-ndk
if: steps.cache.outputs.cache-hit != 'true'
- name: Checkout OCaml
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Configure, build and install Linux-to-Android OCaml
env:
TARGET: x86_64-linux-android21
TOOLDIR: android-ndk-r27b/toolchains/llvm/prebuilt/linux-x86_64/bin
run: |
DIR="$HOME/android/$TOOLDIR"
set -x
# Hack around the fact that pthread_cancel isn't available on Android
# So the result program should _not_ be run with cleanup on exit
# (so no `c=1` in `OCAMLRUNPARAM`)
./configure --prefix="$HOME/cross" --target=$TARGET \
TARGET_LIBDIR="/dummy/directory" \
CC="$DIR/clang --target=$TARGET" \
CPPFLAGS='-Dpthread_cancel=assert' \
AR="$DIR/llvm-ar" \
PARTIALLD="$DIR/ld -r" \
RANLIB="$DIR/llvm-ranlib" \
STRIP="$DIR/llvm-strip" || res=$?
if ! [ "$res" = 0 ]; then cat config.log; exit "$res"; fi
make crossopt -j
make installcross
- name: Show opt.opt configuration
run: |
set -x
$HOME/cross/bin/ocamlopt.opt -config
cat runtime/build_config.h
- name: Cross compile a small program
run: |
printf %s "$EXAMPLE_PROGRAM" > example.ml
set -x
cat example.ml
$HOME/cross/bin/ocamlopt.opt example.ml -o example -verbose
file example
- name: Run example
uses: reactivecircus/android-emulator-runner@v2
with:
api-level: 21
arch: x86_64
disable-animations: true
script: |
adb push example /data/local/tmp/example
adb shell /data/local/tmp/example
21 changes: 10 additions & 11 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -715,9 +715,9 @@ compare:
# The core system has to be rebuilt after bootstrap anyway, so strip ocamlc
# and ocamllex, which means the artefacts should be identical.
mv ocamlc$(EXE) ocamlc.tmp
$(OCAMLRUN) tools/stripdebug -all ocamlc.tmp ocamlc$(EXE)
$(OCAMLRUN) tools/stripdebug$(EXE) -all ocamlc.tmp ocamlc$(EXE)
mv lex/ocamllex$(EXE) ocamllex.tmp
$(OCAMLRUN) tools/stripdebug -all ocamllex.tmp lex/ocamllex$(EXE)
$(OCAMLRUN) tools/stripdebug$(EXE) -all ocamllex.tmp lex/ocamllex$(EXE)
rm -f ocamllex.tmp ocamlc.tmp
@if $(CMPCMD) boot/ocamlc ocamlc$(EXE) \
&& $(CMPCMD) boot/ocamllex lex/ocamllex$(EXE); \
Expand Down Expand Up @@ -745,7 +745,7 @@ promote-cross: promote-common
# Promote the newly compiled system to the rank of bootstrap compiler
# (Runs on the new runtime, produces code for the new runtime)
.PHONY: promote
promote: PROMOTE = $(OCAMLRUN) tools/stripdebug -all
promote: PROMOTE = $(OCAMLRUN) tools/stripdebug$(EXE) -all
promote: promote-common
rm -f boot/ocamlrun$(EXE)
cp runtime/ocamlrun$(EXE) boot/ocamlrun$(EXE)
Expand Down Expand Up @@ -1383,23 +1383,17 @@ runtime/caml/jumptbl.h : runtime/caml/instruct.h
sed -n -e '/^ /s/ \([A-Z]\)/ \&\&lbl_\1/gp' \
-e '/^}/q' > $@

# These are provided as a temporary shim to allow cross-compilation systems
# to supply a host C compiler and different flags and a linking macro.
SAK_CC ?= $(CC)
SAK_CFLAGS ?= $(OC_CFLAGS) $(CFLAGS) $(OC_CPPFLAGS) $(CPPFLAGS)
SAK_LINK ?= $(MKEXE_VIA_CC)

$(SAK): runtime/sak.$(O)
$(V_MKEXE)$(call SAK_LINK,$@,$^)

runtime/sak.$(O): runtime/sak.c runtime/caml/misc.h runtime/caml/config.h
$(V_CC)$(SAK_CC) $(SAK_CFLAGS) $(OUTPUTOBJ)$@ -c $<

C_LITERAL = $(shell $(SAK) encode-C-literal '$(1)')
C_LITERAL = $(shell $(SAK) $(ENCODE_C_LITERAL) '$(1)')

runtime/build_config.h: $(ROOTDIR)/Makefile.config $(SAK)
$(V_GEN)echo '/* This file is generated from $(ROOTDIR)/Makefile.config */' > $@ && \
echo '#define OCAML_STDLIB_DIR $(call C_LITERAL,$(LIBDIR))' >> $@ && \
echo '#define OCAML_STDLIB_DIR $(call C_LITERAL,$(TARGET_LIBDIR))' >> $@ && \
echo '#define HOST "$(HOST)"' >> $@

## Runtime libraries and programs
Expand Down Expand Up @@ -2968,6 +2962,11 @@ endif

include .depend

# Include the cross-compiler recipes only when relevant
ifneq "$(HOST)" "$(TARGET)"
include Makefile.cross
endif

Makefile.config Makefile.build_config: config.status
config.status:
@echo "Please refer to the installation instructions:"
Expand Down
8 changes: 8 additions & 0 deletions Makefile.build_config.in
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,14 @@ OC_DLL_LDFLAGS=@oc_dll_ldflags@

MKEXE_VIA_CC=$(CC) @mkexe_via_cc_ldflags@ @mkexe_via_cc_extra_cmd@

# How to build sak
SAK_CC=@SAK_CC@
SAK_CFLAGS=@SAK_CFLAGS@
SAK_LINK=@SAK_LINK@

# sak command to encode C literals
ENCODE_C_LITERAL=@encode_C_literal@

# Which tool to use to display differences between files
DIFF=@DIFF@
# Which flags to pass to the diff tool
Expand Down
2 changes: 1 addition & 1 deletion Makefile.common
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ endef # OCAML_LIBRARY

# Installing a bytecode executable, with debug information removed
define INSTALL_STRIPPED_BYTE_PROG
$(OCAMLRUN) $(ROOTDIR)/tools/stripdebug $(1) $(1).tmp \
$(OCAMLRUN) $(ROOTDIR)/tools/stripdebug$(EXE) $(1) $(1).tmp \
&& $(INSTALL_PROG) $(1).tmp $(2) \
&& rm $(1).tmp
endef # INSTALL_STRIPPED_BYTE_PROG
Expand Down
Loading

0 comments on commit 1515ed9

Please sign in to comment.