[CI] Fix multiarch builds

Jenkinsfile

@@ -87,6 +87,8 @@ pipeline {
                description: "Math PR to test against. Will check out this PR in the downstream Math repo.")
         string(defaultValue: '', name: 'stanc_flags',
                description: "Pass STANCFLAGS to make/local, default none")
+        booleanParam(defaultValue: false, name: 'build_multiarch_docker', description: 'Build docker image for multiarch builds')
+        string(defaultValue: '', name: 'build_multiarch_docker_tag', description: "Docker tag for the multiarch image")
     }
     options {
         parallelsAlwaysFailFast()
@@ -100,6 +102,7 @@ pipeline {
         GIT_AUTHOR_EMAIL = '[email protected]'
         GIT_COMMITTER_NAME = 'Stan Jenkins'
         GIT_COMMITTER_EMAIL = '[email protected]'
+        MULTIARCH_DOCKER_TAG = 'multiarch-ocaml-4.14'
     }
     stages {
         stage('Verify changes') {
@@ -588,6 +591,51 @@ pipeline {
             }
         }
 
+        stage('Build and push multiarch docker image') {
+            when {
+                beforeAgent true
+                expression {
+                    params.build_multiarch_docker
+                }
+            }
+            agent {
+                dockerfile {
+                    filename 'scripts/docker/builder/Dockerfile'
+                    dir '.'
+                    label 'linux && triqs'
+                    args '--group-add=987 --group-add=980 --group-add=988 --entrypoint=\'\' -v /var/run/docker.sock:/var/run/docker.sock'
+                    additionalBuildArgs  '--build-arg PUID=\$(id -u) --build-arg PGID=\$(id -g)'
+                }
+            }
+            environment { DOCKER_TOKEN = credentials('aada4f7b-baa9-49cf-ac97-5490620fce8a') }
+            steps {
+                script {
+                    retry(3) { checkout scm }
+                    sh '''
+                        docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+                        docker buildx create --name stanc3_builder
+                        docker buildx use stanc3_builder
+
+                        docker login --username stanorg --password "${DOCKER_TOKEN}"
+
+                        cd scripts/docker/multiarch
+
+                        docker buildx build -t stanorg/stanc3:$build_multiarch_docker_tag \
+                        --platform linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/mips64le,linux/s390x \
+                        --build-arg PUID=$(id -u) \
+                        --build-arg PGID=$(id -g) \
+                        --progress=plain --push .
+                    '''
+                }
+            }
+            post {
+                always {
+                    deleteDir()
+                }
+            }
+        }
+
 
         stage('Build binaries') {
             parallel {
@@ -692,17 +740,18 @@ pipeline {
                         dockerfile {
                             filename 'scripts/docker/static/Dockerfile'
                             dir '.'
-                            label 'linux'
+                            label 'linux && triqs'
                             args '--group-add=987 --group-add=980 --group-add=988 --entrypoint=\'\' -v /var/run/docker.sock:/var/run/docker.sock'
                             additionalBuildArgs  '--build-arg PUID=\$(id -u) --build-arg PGID=\$(id -g)'
                         }
                     }
                     steps {
                         dir("${env.WORKSPACE}/linux-mips64el"){
                             cleanCheckout()
+
                             sh """
                                 eval \$(opam env)
-                                bash -x scripts/build_multiarch_stanc3.sh mips64el
+                                bash -x scripts/build_multiarch_stanc3.sh mips64el ${MULTIARCH_DOCKER_TAG}
                             """
 
                             sh "mkdir -p bin && mv `find _build -name stanc.exe` bin/linux-mips64el-stanc"
@@ -725,7 +774,7 @@ pipeline {
                         dockerfile {
                             filename 'scripts/docker/static/Dockerfile'
                             dir '.'
-                            label 'linux'
+                            label 'linux && triqs'
                             args '--group-add=987 --group-add=980 --group-add=988 --entrypoint=\'\' -v /var/run/docker.sock:/var/run/docker.sock'
                             additionalBuildArgs  '--build-arg PUID=\$(id -u) --build-arg PGID=\$(id -g)'
                         }
@@ -735,7 +784,7 @@ pipeline {
                             cleanCheckout()
                             sh """
                                 eval \$(opam env)
-                                bash -x scripts/build_multiarch_stanc3.sh ppc64el
+                                bash -x scripts/build_multiarch_stanc3.sh ppc64el ${MULTIARCH_DOCKER_TAG}
                             """
                             sh "mkdir -p bin && mv `find _build -name stanc.exe` bin/linux-ppc64el-stanc"
                             stash name:'linux-ppc64el-exe', includes:'bin/*'
@@ -756,7 +805,7 @@ pipeline {
                         dockerfile {
                             filename 'scripts/docker/static/Dockerfile'
                             dir '.'
-                            label 'linux'
+                            label 'linux && triqs'
                             args '--group-add=987 --group-add=980 --group-add=988 --entrypoint=\'\' -v /var/run/docker.sock:/var/run/docker.sock'
                             additionalBuildArgs  '--build-arg PUID=\$(id -u) --build-arg PGID=\$(id -g)'
                         }
@@ -766,7 +815,7 @@ pipeline {
                             cleanCheckout()
                             sh """
                                 eval \$(opam env)
-                                bash -x scripts/build_multiarch_stanc3.sh s390x
+                                bash -x scripts/build_multiarch_stanc3.sh s390x ${MULTIARCH_DOCKER_TAG}
                             """
                             sh "mkdir -p bin && mv `find _build -name stanc.exe` bin/linux-s390x-stanc"
                             stash name:'linux-s390x-exe', includes:'bin/*'
@@ -787,7 +836,7 @@ pipeline {
                         dockerfile {
                             filename 'scripts/docker/static/Dockerfile'
                             dir '.'
-                            label 'linux'
+                            label 'linux && triqs'
                             args '--group-add=987 --group-add=980 --group-add=988 --entrypoint=\'\' -v /var/run/docker.sock:/var/run/docker.sock'
                             additionalBuildArgs  '--build-arg PUID=\$(id -u) --build-arg PGID=\$(id -g)'
                         }
@@ -797,7 +846,7 @@ pipeline {
                             cleanCheckout()
                             sh """
                                 eval \$(opam env)
-                                bash -x scripts/build_multiarch_stanc3.sh arm64
+                                bash -x scripts/build_multiarch_stanc3.sh arm64 ${MULTIARCH_DOCKER_TAG}
                             """
                             sh "mkdir -p bin && mv `find _build -name stanc.exe` bin/linux-arm64-stanc"
                             stash name:'linux-arm64-exe', includes:'bin/*'
@@ -818,7 +867,7 @@ pipeline {
                         dockerfile {
                             filename 'scripts/docker/static/Dockerfile'
                             dir '.'
-                            label 'linux'
+                            label 'linux && triqs'
                             args '--group-add=987 --group-add=980 --group-add=988 --entrypoint=\'\' -v /var/run/docker.sock:/var/run/docker.sock'
                             additionalBuildArgs  '--build-arg PUID=\$(id -u) --build-arg PGID=\$(id -g)'
                         }
@@ -828,7 +877,7 @@ pipeline {
                             cleanCheckout()
                             sh """
                                 eval \$(opam env)
-                                bash -x scripts/build_multiarch_stanc3.sh armhf
+                                bash -x scripts/build_multiarch_stanc3.sh armhf ${MULTIARCH_DOCKER_TAG}
                             """
                             sh "mkdir -p bin && mv `find _build -name stanc.exe` bin/linux-armhf-stanc"
                             stash name:'linux-armhf-exe', includes:'bin/*'
@@ -849,7 +898,7 @@ pipeline {
                         dockerfile {
                             filename 'scripts/docker/static/Dockerfile'
                             dir '.'
-                            label 'linux'
+                            label 'linux && triqs'
                             args '--group-add=987 --group-add=980 --group-add=988 --entrypoint=\'\' -v /var/run/docker.sock:/var/run/docker.sock'
                             additionalBuildArgs  '--build-arg PUID=\$(id -u) --build-arg PGID=\$(id -g)'
                         }
@@ -859,7 +908,7 @@ pipeline {
                             cleanCheckout()
                             sh """
                                 eval \$(opam env)
-                                bash -x scripts/build_multiarch_stanc3.sh armel
+                                bash -x scripts/build_multiarch_stanc3.sh armel ${MULTIARCH_DOCKER_TAG}
                             """
                             sh "mkdir -p bin && mv `find _build -name stanc.exe` bin/linux-armel-stanc"
                             stash name:'linux-armel-exe', includes:'bin/*'

scripts/build_multiarch_stanc3.sh

@@ -1,3 +1,4 @@
+
 # Architecture naming isn't consistent between QEMU and Docker, so lookup correct naming
 if [ $1 = "mips64el" ]; then
   export DOCK_ARCH="mips64le"
@@ -19,10 +20,12 @@ elif [ $1 = "s390x" ]; then
   export DOCK_VARIANT=""
 fi
 
+DOCKER_IMAGE_TAG="$2"
+
 # Lookup the sha256 hash for the specified architecture and variant (e.g., v7 for armhf) and strip the enclosing quotations
-SHA=$(skopeo inspect --raw docker://stanorg/stanc3:multiarch-ocaml-4.14 | jq '.manifests | .[] | select(.platform.architecture==env.DOCK_ARCH and .platform.variant==(if env.DOCK_VARIANT == "" then null else env.DOCK_VARIANT end)).digest' | tr -d '"')
+SHA=$(skopeo inspect --raw docker://stanorg/stanc3:${DOCKER_IMAGE_TAG} | jq '.manifests | .[] | select(.platform.architecture==env.DOCK_ARCH and .platform.variant==(if env.DOCK_VARIANT == "" then null else env.DOCK_VARIANT end)).digest' | tr -d '"')
 
 # Register QEMU translation binaries
 docker run --rm --privileged multiarch/qemu-user-static --reset
 
-docker run --privileged -v $(pwd):$(pwd):rw,z stanorg/stanc3:multiarch-ocaml-4.14@$SHA /bin/bash -c "cd $(pwd) && eval \$(opam env) && dune subst && dune build @install --profile static --root=. && chmod -R 777 _build  && chmod -R 777 src && chmod -R 777 test"
+docker run -v $(pwd):$(pwd):rw,z stanorg/stanc3:${DOCKER_IMAGE_TAG}@$SHA /bin/bash -c "cd $(pwd) && eval \$(opam env) && dune subst && dune build @install --profile static --root=."

scripts/docker/builder/Dockerfile

@@ -0,0 +1,24 @@
+# Pull the ubuntu:bionic base image
+FROM ubuntu:bionic
+
+USER root
+
+ARG PUID
+ARG PGID
+
+# Install OS dependencies
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+    curl ca-certificates wget \
+    rsync git build-essential m4 unzip pkg-config libpcre3-dev docker \
+    python3 python3-pip nodejs sudo
+
+RUN curl -sSL https://get.docker.com/ | sh
+
+RUN addgroup -gid ${PGID} jenkins
+RUN adduser --disabled-password --gecos '' --ingroup jenkins --uid ${PUID} jenkins
+RUN usermod -a -G sudo jenkins
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+RUN chown -R jenkins:sudo /usr/local
+
+USER jenkins
+WORKDIR /home/jenkins

scripts/docker/multiarch/Dockerfile

@@ -1,18 +1,26 @@
 # Base image
 FROM debian:buster-20220622-slim
 
-ARG STANC3_BRANCH=update/4.14
-ENV STANC3_BRANCH_ENV=${STANC3_BRANCH}
+USER root
 
-ARG STANC3_ORG=stan-dev
-ENV STANC3_ORG_ENV=${STANC3_ORG}
+ARG PUID
+ARG PGID
 
 # Update repositories and install OS deps
 RUN apt-get update
 RUN apt-get install opam curl bzip2 git tar curl ca-certificates openssl m4 bash -y
 
 RUN update-ca-certificates -f
 
+# Setup jenkins uid/gid
+RUN addgroup -gid ${PGID} jenkins
+RUN adduser --disabled-password --gecos '' --ingroup jenkins --uid ${PUID} jenkins
+RUN usermod -a -G sudo jenkins
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+RUN chown -R jenkins:sudo /usr/local
+
+WORKDIR /qemu-setup
+
 # Identify architecture being built for and cache
 RUN echo $(dpkg --print-architecture) > ./arch
 
@@ -33,28 +41,28 @@ RUN if [ $(cat ./arch) = "arm64" ]; then \
 # Download needed QEMU binary and place in /usr/bin
 RUN curl -L https://github.com/multiarch/qemu-user-static/releases/download/v6.0.0-2/x86_64_qemu-$(cat ./qarch)-static.tar.gz | tar -xzf - -C /usr/bin
 
+RUN chown jenkins /usr/bin/qemu-$(cat ./qarch)-static
+
+USER jenkins
+WORKDIR /home/jenkins
+
 RUN opam init --disable-sandboxing --bare -y
 RUN eval $(opam env) && opam update
 
 # Native-code compilation not available on MIPS, fall back to bytecode
-RUN if [ $(cat ./arch) = "mips64el" ]; then \
+RUN if [ "$(cat /qemu-setup/arch)" = "mips64el" ]; then \
     opam switch create 4.14.1 --packages=ocaml-variants.4.14.1+options,ocaml-option-bytecode-only && opam switch 4.14.1 && opam pin num https://github.com/ocaml/num.git -y; \
   else \
     opam switch create 4.14.1 && opam switch 4.14.1; \
   fi
 
 RUN eval $(opam env) && opam repo add internet https://opam.ocaml.org
 
-# RUN curl https://raw.githubusercontent.com/${STANC3_ORG_ENV}/stanc3/${STANC3_BRANCH_ENV}/scripts/install_build_deps.sh | bash
-
 RUN eval $(opam env) && opam install -y dune
 RUN eval $(opam env) && opam update && opam upgrade
 RUN eval $(opam env) && opam install -y core.v0.16.0
 RUN eval $(opam env) && opam install -y menhir.20230608
 RUN eval $(opam env) && opam install -y ppx_deriving.5.2.1
 RUN eval $(opam env) && opam install -y fmt.0.9.0
 RUN eval $(opam env) && opam install -y yojson.2.1.0
-RUN eval $(opam env)
-
-# Cleanup
-RUN rm ./arch && rm ./qarch
+RUN eval $(opam env)