gh: bump deps and use faasmctl (#19)

* gh: bump deps

* docs: fix readme

* gh: bump planner version

* gh: correct faasm version

* gha: fix error introduced when sed-ing

* gh: properly bump faasm version

* deps: bump again

* nits: format python code

* examples(kernels): fix native mpi kernels compilation

* gha: only build wasm if needed

* gha: use faasmctl

* gha: fix syntax error

* gha: use sudo to install

* gha: faster submodule init

* gha: correct cache key

* faasmctl: bump to 0.19.0

* gha: install with sudo

* gha: clone submodules

* wasm: use the right directory

* gha: fix submodule paths

* gha: more cache fixes

* gha: debug cache miss

* data: not rely on lammps submodule being checked out

* gha: attempt to fix mpi kernels

* nits: self-review

Author: csegarragonz

.github/workflows/tests.yml

@@ -10,3 +10,6 @@ build
 
 # WASM files
 wasm
+
+# Faasm deployment config
+faasm.ini

.gitignore

@@ -1 +1 @@
-0.9.9
+0.15.0

FAASM_VERSION

@@ -84,7 +84,7 @@ inv docker.build -c build -c run --nocache --push
 ## Adding a new application
 
 To add a new application, you first need to cross-compile it to WebAssembly.
-You can check the [`tasks/`]](./tasks) folder for examples of how we do it for
+You can check the [`tasks/`](./tasks) folder for examples of how we do it for
 existing applications. Most importantly, you will have to inidicate the right
 sysroot, and pass the environment variables that we read from `faasmtools`.
 

README.md

@@ -0,0 +1,46 @@
+# 3d Lennard-Jones melt, thermostatted by fix controller
+
+units           lj
+atom_style      atomic
+boundary        p p m
+processors      * * 1
+lattice         fcc 0.8442
+region          box block 0 10 0 10 -4 14
+region          slab block 0 10 0 10 0 10
+create_box      1 box
+create_atoms    1 region slab
+mass            1 1.0
+
+velocity        all create 1.44 87287 loop geom
+
+pair_style      lj/cut 2.5
+pair_coeff      1 1 1.0 1.0 2.5
+
+neighbor        0.3 bin
+neigh_modify    delay 2 every 2 check yes
+
+fix             1 all nve
+fix             3 all langevin 1.5 1.5 0.5 412513
+
+variable        zhi internal $(10.5*zlat)
+variable        kwall equal 20.0
+fix             2 all wall/harmonic zlo -0.5 ${kwall} 0.0 2.5 zhi v_zhi ${kwall} 0.0 2.5
+
+variable        pzz equal pzz
+fix             ave all ave/time 10 10 100 v_pzz
+# equilibrate
+
+thermo_style    custom  step temp pxx pyy pzz f_ave v_zhi
+thermo           500
+run             2500
+
+# use time averaged pressure for control
+#fix             10 all controller 100 -0.05 20.0 0.0 0.0 f_ave 1.5 zhi
+
+# use instantaneous pressure for control
+fix             10 all controller 100 -0.05 50.0 0.0 0.0 v_pzz 1.5 zhi
+
+# run with controller
+
+run             5000
+

cpp

@@ -5,7 +5,7 @@ services:
     image: faasm.azurecr.io/redis:${EXAMPLES_RUN_VERSION}
 
   planner:
-    image: faasm.azurecr.io/planner:0.4.4
+    image: faasm.azurecr.io/planner:0.10.0
     environment:
       - LOG_LEVEL=info
       - PLANNER_PORT=8081

data/in.controller.wall

@@ -1,3 +1,4 @@
-black==22.3.0
-flake8==4.0.1
-invoke==1.7.1
+black>=22.3.0
+faasmctl>=0.19.0
+flake8>=4.0.1
+invoke>=1.7.1

docker-compose.yml

@@ -1,49 +1,16 @@
-from faasmtools.build import FAASM_LOCAL_DIR
 from invoke import task
 from os import makedirs
-from os.path import dirname, exists, join
+from os.path import dirname, exists
 from shutil import copyfile
-from tasks.env import EXAMPLES_DIR, PROJ_ROOT
-
-DATA_BASE_DIR = join(FAASM_LOCAL_DIR, "shared")
-DATA_HOST_DIR = join(PROJ_ROOT, "data")
+from tasks.env import EXAMPLES_DATA_FILES
 
 
 @task(default=True)
 def prepare(ctx):
     """
     Prepare data files to test WASM examples
     """
-    data_files = [
-        [
-            join(DATA_HOST_DIR, "faasm_logo.png"),
-            join(DATA_BASE_DIR, "im", "sample_image.png"),
-        ],
-        [
-            join(
-                EXAMPLES_DIR,
-                "lammps",
-                "examples",
-                "controller",
-                "in.controller.wall",
-            ),
-            join(DATA_BASE_DIR, "lammps-data", "in.controller.wall"),
-        ],
-        [
-            join(DATA_HOST_DIR, "ffmpeg_video.mp4"),
-            join(DATA_BASE_DIR, "ffmpeg", "sample_video.mp4"),
-        ],
-        [
-            join(DATA_HOST_DIR, "sample_model.tflite"),
-            join(DATA_BASE_DIR, "tflite", "sample_model.tflite"),
-        ],
-        [
-            join(DATA_HOST_DIR, "grace_hopper.bmp"),
-            join(DATA_BASE_DIR, "tflite", "grace_hopper.bmp"),
-        ],
-    ]
-
-    for p in data_files:
+    for p in EXAMPLES_DATA_FILES:
         path_src = p[0]
         path_dst = p[1]
 

examples/Kernels

@@ -1,6 +1,9 @@
+from faasmtools.build import FAASM_LOCAL_DIR
 from faasmtools.docker import ACR_NAME
 from faasmtools.env import get_version as get_cpp_version
 from os.path import dirname, abspath, join
+from re import search as re_search
+from subprocess import run
 
 PROJ_ROOT = dirname(dirname(abspath(__file__)))
 DOCKER_ROOT = join(PROJ_ROOT, "docker")
@@ -14,6 +17,32 @@
 EXAMPLES_RUN_IMAGE_NAME = "{}/examples-run".format(ACR_NAME)
 EXAMPLES_RUN_DOCKERFILE = join(DOCKER_ROOT, "run.dockerfile")
 
+# Shared files data
+EXAMPLES_DATA_BASE_DIR = join(FAASM_LOCAL_DIR, "shared")
+EXAMPLES_DATA_HOST_DIR = join(PROJ_ROOT, "data")
+EXAMPLES_DATA_FILES = [
+    [
+        join(EXAMPLES_DATA_HOST_DIR, "faasm_logo.png"),
+        join(EXAMPLES_DATA_BASE_DIR, "im", "sample_image.png"),
+    ],
+    [
+        join(EXAMPLES_DATA_HOST_DIR, "in.controller.wall"),
+        join(EXAMPLES_DATA_BASE_DIR, "lammps-data", "in.controller.wall"),
+    ],
+    [
+        join(EXAMPLES_DATA_HOST_DIR, "ffmpeg_video.mp4"),
+        join(EXAMPLES_DATA_BASE_DIR, "ffmpeg", "sample_video.mp4"),
+    ],
+    [
+        join(EXAMPLES_DATA_HOST_DIR, "sample_model.tflite"),
+        join(EXAMPLES_DATA_BASE_DIR, "tflite", "sample_model.tflite"),
+    ],
+    [
+        join(EXAMPLES_DATA_HOST_DIR, "grace_hopper.bmp"),
+        join(EXAMPLES_DATA_BASE_DIR, "tflite", "grace_hopper.bmp"),
+    ],
+]
+
 
 def get_submodule_version(submodule):
     """
@@ -48,6 +77,33 @@ def get_faasm_version():
     return version
 
 
+def get_faabric_version(old_faasm_ver, new_faasm_ver):
+    """
+    Get the faabric version by `wget`-ing the FAASM version file for the tagged
+    Faasm version
+    """
+
+    def do_get_ver(faasm_ver):
+        tmp_file = "/tmp/faabric_version"
+        wget_cmd = [
+            "wget",
+            "-O {}".format(tmp_file),
+            "https://raw.githubusercontent.com/faasm/faasm/v{}/.env".format(
+                faasm_ver
+            ),
+        ]
+        wget_cmd = " ".join(wget_cmd)
+        out = run(wget_cmd, shell=True, capture_output=True)
+        assert out.returncode == 0
+
+        with open(tmp_file, "r") as fh:
+            ver = re_search(r"planner:([0-9\.]*)", fh.read()).groups(1)[0]
+
+        return ver
+
+    return do_get_ver(old_faasm_ver), do_get_ver(new_faasm_ver)
+
+
 def get_version(name="build"):
     """
     Get version for the examples repository

python

@@ -3,12 +3,14 @@
 from tasks.env import (
     EXAMPLES_BUILD_IMAGE_NAME,
     PROJ_ROOT,
+    get_faabric_version,
     get_faasm_version,
     get_version,
 )
 
 VERSIONED_FILES = {
     "faasm": ["FAASM_VERSION", ".github/workflows/tests.yml"],
+    "faabric": ["docker-compose.yml", ".github/workflows/tests.yml"],
     "cpp": [".github/workflows/tests.yml"],
     "python": [".github/workflows/tests.yml"],
 }
@@ -52,16 +54,19 @@ def bump(ctx, submodule, ver=None):
         if ver:
             new_ver = ver
         else:
-            # Just bump the last minor version part
-            new_ver_parts = old_ver.split(".")
-            new_ver_minor = int(new_ver_parts[-1]) + 1
-            new_ver_parts[-1] = str(new_ver_minor)
-            new_ver = ".".join(new_ver_parts)
+            raise RuntimeError("Must provide a version with --ver flag!")
 
         # Replace version in all files
         for f in VERSIONED_FILES["faasm"]:
             sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f)
             run(sed_cmd, shell=True, check=True)
+
+        # Also update the planner version accordingly
+        old_fabric_ver, new_faabric_ver = get_faabric_version(old_ver, new_ver)
+        for f in VERSIONED_FILES["faabric"]:
+            sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f)
+            run(sed_cmd, shell=True, check=True)
+
     else:
         new_ver = get_version("build")
         grep_cmd = "grep '{}' .github/workflows/tests.yml".format(

requirements.txt

@@ -1,9 +1,15 @@
 from distutils.dir_util import copy_tree
+from faasmctl.util.upload import upload_file, upload_wasm
 from invoke import task
-from os import makedirs
+from os import listdir, makedirs
 from os.path import exists, join
 from shutil import copyfile, rmtree
-from tasks.env import EXAMPLES_DIR, WASM_DIR
+from tasks.env import (
+    EXAMPLES_DIR,
+    EXAMPLES_DATA_FILES,
+    EXAMPLES_DATA_BASE_DIR,
+    WASM_DIR,
+)
 
 
 @task(default=True)
@@ -33,3 +39,26 @@ def copy(ctx, clean=False):
     wasm_dirs = [join(EXAMPLES_DIR, "Kernels", "build", "wasm")]
     for wasm_dir in wasm_dirs:
         copy_tree(wasm_dir, WASM_DIR)
+
+
+@task
+def upload(ctx):
+    """
+    Upload all WASM and shared data to a Faasm cluster
+    """
+    if not exists(WASM_DIR):
+        print("Expected WASM files to be available in {}".format(WASM_DIR))
+        raise RuntimeError("WASM directory not found!")
+
+    # Iterate over all dirs and upload
+    for user in listdir(WASM_DIR):
+        for name in listdir(join(WASM_DIR, user)):
+            wasm_path = join(WASM_DIR, user, name, "function.wasm")
+            if exists(wasm_path):
+                upload_wasm(user, name, wasm_path)
+
+    # Iterate over all shared data files and upload
+    for data_file in EXAMPLES_DATA_FILES:
+        host_path = data_file[0]
+        faasm_path = data_file[1].removeprefix(EXAMPLES_DATA_BASE_DIR + "/")
+        upload_file(host_path, faasm_path)