LLVM 18 (#34)

* gh: bump cpp and python submodules

* tasks: change usage of dict for getter method

* gha: bump cpp and python tags

* tasks(docker): remove remanents of  service and use base image

* examples(kernels): fix compilation with llvm 17

* lulesh: fix cross-compilation with llvm 17

* examples(tensorflow): fix compilation for llvm 17

* examples(ffmpeg): fix func ompilation

* examples(imagemagick): cross-compilation works with llvm 17

* cpp: bump to latest version

* gha: fix release gha workflow

* examples(kernels): bump submodule commit

* examples(kernels): fix compilation of native kernels

* func(tf): fix compilation

* docker: fix dockerfile

* nits: self-rewiew

* gha: run omp kernels with wamr

* examples(kernels): prepare omp kernels to run with wamr

* cpp: bump to latest commit

* gh: bump cpp and python after merge to main

* gha: debug wavm failure

* gha: print upload logs on failure

* gha: change how we cache wasm

* gha: more wavm investigation

* gha: fix wavm wasm upload issue

* gha: comment out another wavm app

Author: csegarragonz

.github/workflows/release.yml

@@ -26,10 +26,8 @@ jobs:
           submodules: true
       - name: "Set version variables"
         run: |
-          FAASM_VER=$(cat FAASM_VERSION)
           CPP_VER=$(cat cpp/VERSION)
           PYTHON_VER=$(cat python/VERSION)
-          echo "FAASM_VERSION=$FAASM_VER" >> $GITHUB_ENV
           echo "CPP_VERSION=$CPP_VER" >> $GITHUB_ENV
           echo "PYTHON_VERSION=$PYTHON_VER" >> $GITHUB_ENV
           echo "EXAMPLES_VERSION=${CPP_VER}_${PYTHON_VER}" >> $GITHUB_ENV
@@ -43,6 +41,16 @@ jobs:
           registry: faasm.azurecr.io
           username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
           password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
+      - name: "Build and push examples-base image"
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          file: docker/base.dockerfile
+          build-args: |
+            OPENMPI_VERSION=4.1.0
+            OPENMPI_VERSION_NOPATCH=4.1
+          context: .
+          tags: faasm.azurecr.io/examples-base:${{ env.EXAMPLES_VERSION }}
       - name: "Build and push examples-build image"
         uses: docker/build-push-action@v4
         with:

.github/workflows/tests.yml

@@ -33,40 +33,32 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     outputs:
-      needs-wasm: ${{ (steps.filter.outputs.examples-changed == 'true') || (steps.wasm-cache.outputs.cache-hit != 'true') }}
+      needs-wasm: ${{ steps.wasm-cache.outputs.cache-hit != 'true' }}
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v4
-      # Check if any of the submodules have been modified
-      - uses: dorny/paths-filter@v3
-        id: filter
         with:
-          filters: |
-            examples-changed:
-              - './examples/**'
+          submodules: true
       # Check if we have a WASM cache hit
-      - name: "Get CPP/Python commits"
+      - name: "Get digest of all submodules"
         id: submodule-commit
         run: |
-          git submodule update --init ./cpp
-          git submodule update --init ./python
           sudo apt install -y zstd
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          echo "cpp-commit=$(git submodule status ./cpp | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
-          echo "py-commit=$(git submodule status ./python | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
+          echo "digest=$(git submodule status | awk '{ print $1; }' | md5sum | cut -d' ' -f1)" >> $GITHUB_OUTPUT
       - uses: actions/cache/restore@v4
         id: wasm-cache
         with:
           path: ./wasm
-          key: wasm-${{ steps.submodule-commit.outputs.cpp-commit }}-${{ steps.submodule-commit.outputs.py-commit }}
+          key: wasm-${{ steps.submodule-commit.outputs.digest }}
           lookup-only: true
 
   build-examples:
     needs: needs-build
     if: ${{ needs.needs-build.outputs.needs-wasm == 'true' }}
     runs-on: ubuntu-latest
     container:
-      image: faasm.azurecr.io/examples-build:0.3.1_0.3.1
+      image: faasm.azurecr.io/examples-build:0.4.0_0.4.0
       credentials:
         username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
         password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
@@ -100,8 +92,7 @@ jobs:
         run: |
           apt install -y zstd
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          echo "cpp-commit=$(git submodule status ./cpp | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
-          echo "py-commit=$(git submodule status ./python | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
+          echo "digest=$(git submodule status | awk '{ print $1; }' | md5sum | cut -d' ' -f1)" >> $GITHUB_OUTPUT
           # Also move to a different path to restore from
           mv /usr/local/faasm/wasm ./wasm
       # If we are here we _always_ want to overwrite the cache
@@ -110,7 +101,7 @@ jobs:
         if: always()
         with:
           path: ./wasm
-          key: wasm-${{ steps.submodule-commit.outputs.cpp-commit }}-${{ steps.submodule-commit.outputs.py-commit }}
+          key: wasm-${{ steps.submodule-commit.outputs.digest }}
 
   run-examples-faasmctl:
     needs: [build-examples]
@@ -128,9 +119,8 @@ jobs:
       run:
         shell: bash
     env:
-      CONAN_CACHE_MOUNT_SOURCE: ~/.conan
       FAASM_INI_FILE: ./faasm.ini
-      FAASM_VERSION: 0.21.2
+      FAASM_VERSION: 0.22.0
       FAASM_WASM_VM: ${{ matrix.faasm_wasm_vm }}
     steps:
       - uses: csegarragonz/set-compose-version-action@main
@@ -149,6 +139,7 @@ jobs:
           remove-docker-images: 'true'
           remove-dotnet: 'true'
           remove-haskell: 'true'
+
       - name: "Checkout code"
         uses: actions/checkout@v4
       - name: "Install faasmctl"
@@ -179,7 +170,11 @@ jobs:
           key: wasm-${{ steps.submodule-commit.outputs.cpp-commit }}-${{ steps.submodule-commit.outputs.py-commit }}
           fail-on-cache-miss: true
       - name: "Upload all WASM files to the cluster"
+        id: wasm-upload
         run: ./bin/inv_wrapper.sh wasm.upload
+      - name: "Print logs if wasm-upload fails"
+        if: failure() && (steps.wasm-upload.outcome == 'failure')
+        run: faasmctl logs -s upload
       - name: "Run MPI kernels"
         timeout-minutes: 2
         run: |
@@ -205,13 +200,12 @@ jobs:
           faasmctl invoke kernels-mpi p2p --cmdline '10 1024 1024' --mpi-world-size 4
           faasmctl invoke kernels-mpi transpose --cmdline '10 1024 32' --mpi-world-size 4
       - name: "Run OpenMP kernels"
-        if: "contains(env.FAASM_WASM_VM, 'wavm')"
+        if: "contains(env.FAASM_WASM_VM, 'wamr')"
         timeout-minutes: 2
         run: |
           faasmctl invoke kernels-omp global --cmdline '2 10 1024'
           faasmctl invoke kernels-omp p2p --cmdline '2 10 1024 1024'
-          # The sparse OpenMP kernel fails during the result integrity checks
-          # faasmctl invoke kernels-omp sparse --cmdline '2 10 10 5'
+          faasmctl invoke kernels-omp sparse --cmdline '2 10 10 5'
           # The transpose OpenMP kernel can not be cross-compiled due to problems
           # with the static_for signature and symbol definition
           # faasmctl invoke kernels-omp transpose --cmdline '10 1024 32'
@@ -254,10 +248,11 @@ jobs:
           faasmctl invoke polybench poly_jacobi-1d
           faasmctl invoke polybench poly_jacobi-2d
           faasmctl invoke polybench poly_seidel-2d
-      - name: "Run ImageMagick"
-        if: "contains(env.FAASM_WASM_VM, 'wavm')"
-        timeout-minutes: 1
-        run: faasmctl invoke imagemagick main --cmdline 'faasm://im/sample_image.png -flip faasm://im/image_out.png'
+          # TODO(wamr-tf): TF Check broken with WAVM, uncomment when WAMR supports it
+          #       - name: "Run ImageMagick"
+          #         if: "contains(env.FAASM_WASM_VM, 'wamr')"
+          #         timeout-minutes: 1
+          #         run: faasmctl invoke imagemagick main --cmdline 'faasm://im/sample_image.png -flip faasm://im/image_out.png'
       - name: "Run LAMMPS"
         timeout-minutes: 1
         run: faasmctl invoke lammps main --cmdline '-in faasm://lammps-data/in.controller.wall' --mpi-world-size 4
@@ -268,15 +263,18 @@ jobs:
       - name: "Run LAMMPS migration-net binary"
         timeout-minutes: 1
         run: faasmctl invoke lammps migration-net --cmdline '-in faasm://lammps-data/in.controller.wall' --mpi-world-size 4 --input-data '1 2 10 2'
-          # 08/12/2023 - LULESH is broken with a memory leak
-          #       - name: "Run LULESH"
-          #         timeout-minutes: 1
-          #         if: "contains(env.FAASM_WASM_VM, 'wavm')"
-          #         run: faasmctl invoke lulesh main --cmdline '-i 5 -s 20 -r 11 -c 1 -b 1' --input-data '4'
-      - name: "Run TFlite check"
+      - name: "Run LULESH"
         timeout-minutes: 1
-        if: "contains(env.FAASM_WASM_VM, 'wavm')"
-        run: faasmctl invoke tf check
+        if: "contains(env.FAASM_WASM_VM, 'wamr')"
+        run: faasmctl invoke lulesh main --cmdline '-i 5 -s 20 -r 11 -c 1 -b 1' --input-data '4'
+          # TODO(wamr-tf): TF Check broken with WAVM, uncomment when WAMR supports it
+          #       - name: "Run TFlite check"
+          #         timeout-minutes: 1
+          #         if: "contains(env.FAASM_WASM_VM, 'wamr')"
+          #         run: faasmctl invoke tf check
       - name: "Run FFmpeg check"
         timeout-minutes: 1
         run: faasmctl invoke ffmpeg check
+      - name: "Print logs in case of failure"
+        if: failure()
+        run: faasmctl logs -s worker

bin/workon.sh

@@ -49,7 +49,6 @@ complete -F _complete_invoke -o default invoke inv
 # Environment vars
 # ----------------------------
 
-export FAASM_VERSION=$(cat ${PROJ_ROOT}/FAASM_VERSION)
 export CPP_VERSION=$(cat ${PROJ_ROOT}/cpp/VERSION)
 export PYTHON_VERSION=$(cat ${PROJ_ROOT}/python/VERSION)
 
@@ -62,7 +61,6 @@ export PS1="(faasm-examples) $PS1"
 echo ""
 echo "----------------------------------"
 echo "Faasm Examples CLI"
-echo "Faasm Version: ${FAASM_VERSION}"
 echo "CPP Version: ${CPP_VERSION}"
 echo "Python Version: ${PYTHON_VERSION}"
 echo "----------------------------------"

cpp

@@ -0,0 +1,20 @@
+FROM ubuntu:22.04 as base
+
+RUN apt update \
+    && apt install -y \
+        bzip2 \
+        gcc \
+        g++ \
+        make \
+        wget
+
+# Download and build MPI
+ARG OPENMPI_VERSION
+ARG OPENMPI_VERSION_NOPATCH
+RUN mkdir -p /tmp \
+    && cd /tmp \
+    && wget https://download.open-mpi.org/release/open-mpi/v${OPENMPI_VERSION_NOPATCH}/openmpi-${OPENMPI_VERSION}.tar.bz2 \
+    && tar xf openmpi-${OPENMPI_VERSION}.tar.bz2 \
+    && cd /tmp/openmpi-${OPENMPI_VERSION} \
+    && ./configure --prefix=/usr/local \
+    && make -j `nproc`

docker/base.dockerfile

@@ -1,24 +1,6 @@
 ARG CPP_VERSION
-FROM ubuntu:22.04 as base
-
-RUN apt update \
-    && apt install -y \
-        bzip2 \
-        gcc \
-        g++ \
-        make \
-        wget
-
-# Download and build MPI
-RUN mkdir -p /tmp \
-    && cd /tmp \
-    && wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.bz2 \
-    && tar xf openmpi-4.1.0.tar.bz2 \
-    && cd /tmp/openmpi-4.1.0 \
-    && ./configure --prefix=/usr/local \
-    && make -j `nproc`
-
-ARG CPP_VERSION
+ARG EXAMPLES_VERSION
+FROM faasm.azurecr.io/examples-base:${EXAMPLES_VERSION} as base
 FROM faasm.azurecr.io/cpp-sysroot:${CPP_VERSION}
 
 SHELL ["/bin/bash", "-c"]
@@ -36,7 +18,7 @@ RUN cd /tmp/openmpi-4.1.0 \
 # Install OpenMP
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt update \
-    && apt install -y libomp-13-dev
+    && apt install -y libomp-17-dev
 
 # Fetch the code and update submodules
 ARG EXAMPLES_VERSION
@@ -53,6 +35,7 @@ RUN mkdir -p code \
     && git submodule update --init -f examples/Kernels \
     && git submodule update --init -f examples/lammps \
     && git submodule update --init -f examples/lammps-migration \
+    && git submodule update --init -f examples/lammps-migration-net \
     && git submodule update --init -f examples/LULESH \
     && git submodule update --init -f examples/libpng \
     && git submodule update --init -f examples/polybench \
@@ -63,28 +46,28 @@ RUN cd /code/examples \
     && ./bin/create_venv.sh \
     && source venv/bin/activate \
     # Build the native versions of the examples that support it
-    && inv \
-        kernels --native \
-        lammps --native \
-        lammps --migration --native \
-        lulesh --native \
-        polybench --native \
-    && inv \
-        ffmpeg \
-        # ImageMagick needs libpng
-        libpng imagemagick \
-        kernels \
-        lammps \
-        lammps --migration \
-        lulesh \
-        polybench \
-        tensorflow \
+    && inv kernels --native \
+    && inv lammps --native \
+    && inv lammps --migration --native \
+    && inv lammps --migration-net --native \
+    && inv lulesh --native \
+    && inv polybench --native \
+    # Build the WASM applications
+    && inv ffmpeg \
+    # ImageMagick needs libpng
+    && inv libpng imagemagick \
+    && inv kernels \
+    && inv lammps \
+    && inv lammps --migration \
+    && inv lammps --migration-net \
+    && inv lulesh \
+    && inv polybench \
+    && inv tensorflow \
     # These demo functions link with the cross-compiled static libraries
-    && inv \
-        func ffmpeg check \
-        func lammps chain \
-        func mpi migrate \
-        func tf check
+    && inv func ffmpeg check \
+    && inv func lammps chain \
+    && inv func mpi migrate \
+    && inv func tf check
 
 # Prepare bashrc
 WORKDIR /code/examples

docker/build.dockerfile

@@ -2,15 +2,15 @@ set(FAASM_USER ffmpeg)
 
 # FFmpeg build
 set(FFMPEG_INCLUDE_DIRS
-    ${CMAKE_SYSROOT}/include/libavcodec
-    ${CMAKE_SYSROOT}/include/libavformat
+    $ENV{FAASM_WASM_HEADER_INSTALL_DIR}/libavcodec
+    $ENV{FAASM_WASM_HEADER_INSTALL_DIR}/libavformat
 )
 set(FFMPEG_LIBS
-    ${CMAKE_SYSROOT}/lib/wasm32-wasi/libavcodec.a
-    ${CMAKE_SYSROOT}/lib/wasm32-wasi/libavformat.a
-    ${CMAKE_SYSROOT}/lib/wasm32-wasi/libavutil.a
-    ${CMAKE_SYSROOT}/lib/wasm32-wasi/libswscale.a
-    ${CMAKE_SYSROOT}/lib/wasm32-wasi/libswresample.a
+    $ENV{FAASM_WASM_LIB_INSTALL_DIR}/libavcodec.a
+    $ENV{FAASM_WASM_LIB_INSTALL_DIR}/libavformat.a
+    $ENV{FAASM_WASM_LIB_INSTALL_DIR}/libavutil.a
+    $ENV{FAASM_WASM_LIB_INSTALL_DIR}/libswscale.a
+    $ENV{FAASM_WASM_LIB_INSTALL_DIR}/libswresample.a
 )
 
 faasm_example_func(check check.cpp)

examples/ImageMagick

@@ -1,12 +1,16 @@
 set(FAASM_USER tf)
 
-set(TF_LIB ${CMAKE_SYSROOT}/lib/wasm32-wasi/libtensorflow-lite.a)
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --target=$ENV{FAASM_WASM_TRIPLE} -pthread")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --target=$ENV{FAASM_WASM_TRIPLE} -pthread")
+set(CMAKE_EXE_LINKER_FLAGS "$ENV{FAASM_WASM_EXE_LINKER_FLAGS}")
+
+set(TF_LIB libtensorflow-lite.a)
 
 faasm_example_func(check check.cpp)
 
 set(TF_HEADERS
-    ${CMAKE_SYSROOT}/include/tensorflow
-    ${CMAKE_SYSROOT}/include/tensorflow/tensorflow/lite/tools/make/downloads/flatbuffers/include
+    $ENV{FAASM_WASM_HEADER_INSTALL_DIR}/tensorflow
+    $ENV{FAASM_WASM_HEADER_INSTALL_DIR}/tensorflow/tensorflow/lite/tools/make/downloads/flatbuffers/include
 )
 
 target_include_directories(tf_check PUBLIC ${TF_HEADERS})

examples/Kernels

@@ -1,5 +1,5 @@
 black>=22.3.0
-faasmctl>=0.29.0
+faasmctl>=0.32.0
 flake8>=4.0.1
 invoke>=1.7.1
 PyYAML>=6.0.1

examples/LULESH

@@ -13,10 +13,12 @@
 
 
 @task(default=True)
-def cli(ctx, service, clean=False):
+def cli(ctx, clean=False):
     """
-    Get a shell into one of the containers: `build` or `run`
+    Get a shell into the examples build container
     """
+    service = "build"
+
     if clean:
         # Clean existing build
         if exists(DEV_FAASM_LOCAL):