Added RunTime set of images that are slimmer and available in jupyter

Signed-off-by: Aloys Baillet <[email protected]>

.github/workflows/docker-builds.yml

@@ -24,12 +24,12 @@ jobs:
         group: [img-common-1] # Extra groups come from the include next
         include:
           - group: img-common-1
-            type: IMAGE
+            type: CI_IMAGE
             name: common
             version: 1
             test: common
           - group: img-common-2
-            type: IMAGE
+            type: CI_IMAGE
             name: common
             version: 2
             test: common
@@ -46,31 +46,31 @@ jobs:
             version: 2021
             test: ""
           - group: img-base-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: base
             common_version: 2
             version: 2021
             test: base
           - group: img-vfx1-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: vfx1
             common_version: 2
             version: 2021
             test: openvdb
           - group: img-vfx2-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: vfx2
             common_version: 2
             version: 2021
             test: ""
           - group: img-usd-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: usd
             common_version: 2
             version: 2021
             test: ""
           - group: img-vfxall-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: vfxall
             common_version: 2
             version: 2021
@@ -116,11 +116,11 @@ jobs:
           --source-branch $GITHUB_REF \
           --verbose \
           build \
-          --ci-image-type IMAGE \
+          --ci-image-type CI_IMAGE \
           --group common \
           --version ${{ matrix.common_version }}
       name: Optionally build local ci-common Docker Image for downstream images
-      if: ${{ matrix.name != 'common' && matrix.type == 'IMAGE' }}
+      if: ${{ matrix.name != 'common' && matrix.type == 'CI_IMAGE' }}
 
     - run: |
         set -ex

CONTRIBUTING.md

@@ -284,7 +284,7 @@ Example use: just build a single package for testing:
 # Build and push USD package to aswftesting
 aswfdocker --verbose build -t PACKAGE --group vfx --version 2019 --target usd --push
 # Build and push ci-vfxall image to aswftesting
-aswfdocker --verbose build -t IMAGE --group vfx --version 2019 --target vfxall --push
+aswfdocker --verbose build -t CI_IMAGE --group vfx --version 2019 --target vfxall --push
 ```
 
 ### Migrate

README.md

@@ -116,15 +116,15 @@ but it is recommended to speed up large builds.
 
 To build all images (very unlikely to succeed unless run on a very very powerful machine!):
 ```bash
-aswfdocker --verbose build -t IMAGE
+aswfdocker --verbose build -t CI_IMAGE
 ```
 
 To build a single image:
 ```bash
 # First list the available CI images to know which package belong to which "group":
 aswfdocker images
 # Then run the build
-aswfdocker --verbose build -t IMAGE --group vfx1 --version 2019 --target openexr
+aswfdocker --verbose build -t CI_IMAGE --group vfx1 --version 2019 --target openexr
 # Or the simpler but less flexible syntax:
 aswfdocker build -n aswftesting/ci-openexr:2019
 ```

python/aswfdocker/cli/aswfdocker.py

@@ -257,15 +257,14 @@ def packages():
 
 @cli.command()
 def images():
-    """Lists all known ci images in this format: IMAGEGROUP/ci-IMAGE:VERSION
+    """Lists all known ci images in this format: IMAGEGROUP/ci-CI_IMAGE:VERSION
     """
-    for group, images in constants.GROUPS[constants.ImageType.IMAGE].items():
-        for image in images:
-            image_name = utils.get_image_name(constants.ImageType.IMAGE, image)
-            for version in index.Index().iter_versions(
-                constants.ImageType.IMAGE, image
-            ):
-                click.echo(f"{group}/{image_name}:{version}")
+    for image_type in (constants.ImageType.CI_IMAGE, constants.ImageType.RT_IMAGE):
+        for group, images in constants.GROUPS[image_type].items():
+            for image in images:
+                image_name = utils.get_image_name(image_type, image)
+                for version in index.Index().iter_versions(image_type, image):
+                    click.echo(f"{group}/{image_name}:{version}")
 
 
 @cli.command()

python/aswfdocker/constants.py

@@ -9,7 +9,8 @@
 
 
 class ImageType(enum.Enum):
-    IMAGE = "image"
+    CI_IMAGE = "ci-image"
+    RT_IMAGE = "rt-image"
     PACKAGE = "package"
 
 
@@ -24,14 +25,19 @@ class ImageType(enum.Enum):
         "vfx2": ["opensubdiv", "ptex", "openvdb", "usd", "otio"],
         "vfx3": ["partio", "osl"],
     },
-    ImageType.IMAGE: {
+    ImageType.CI_IMAGE: {
         "common": ["common"],
         "base": ["base"],
         "vfx1": ["openexr", "openvdb", "opencue"],
         "vfx2": ["ocio", "osl", "otio"],
         "usd": ["usd"],
         "vfxall": ["vfxall"],
     },
+    ImageType.RT_IMAGE: {
+        "base": ["base"],
+        "vfx1": ["vfxall"],
+        "vfx2": ["vfxall-jupyter"],
+    },
 }
 
 VERSION_INFO = {
@@ -95,4 +101,4 @@ class ImageType(enum.Enum):
     f"https://github.com/{MAIN_GITHUB_ASWF_ORG}/{MAIN_GITHUB_REPO_NAME}"
 )
 
-IMAGE_NAME_REGEX = r"(refs/tags/)?(?P<org>[a-z]+)/ci-(?P<package>package\-)?(?P<image>[a-z0-9]+)[:/](?P<version>[0-9\.]+)"
+IMAGE_NAME_REGEX = r"(refs/tags/)?(?P<org>[a-z]+)/(?P<ci>ci-)?(?P<rt>rt-)?(?P<package>package\-)?(?P<image>[a-z0-9\-]+)[:/](?P<version>[0-9\.]+)"

python/aswfdocker/index.py

@@ -24,7 +24,9 @@ def __init__(self):
     def _get_key(self, image_type: constants.ImageType):
         if image_type == constants.ImageType.PACKAGE:
             return "ci-packages"
-        return "ci-images"
+        if image_type == constants.ImageType.CI_IMAGE:
+            return "ci-images"
+        return "rt-images"
 
     def iter_images(self, image_type: constants.ImageType):
         """

python/aswfdocker/tests/test_builder.py

@@ -73,12 +73,12 @@ def test_image_base_2019_dict(self):
             groupinfo.GroupInfo(
                 names=["base"],
                 versions=["2019"],
-                type_=constants.ImageType.IMAGE,
+                type_=constants.ImageType.CI_IMAGE,
                 targets=[],
             ),
         )
         base_version = list(
-            index.Index().iter_versions(constants.ImageType.IMAGE, "base")
+            index.Index().iter_versions(constants.ImageType.CI_IMAGE, "base")
         )[1]
         self.assertEqual(
             b.make_bake_dict(),
@@ -118,12 +118,12 @@ def test_image_base_2019_2020_dict(self):
             groupinfo.GroupInfo(
                 names=["base"],
                 versions=["2019", "2020"],
-                type_=constants.ImageType.IMAGE,
+                type_=constants.ImageType.CI_IMAGE,
                 targets=[],
             ),
         )
         base_versions = list(
-            index.Index().iter_versions(constants.ImageType.IMAGE, "base")
+            index.Index().iter_versions(constants.ImageType.CI_IMAGE, "base")
         )
         self.assertEqual(
             b.make_bake_dict(),
@@ -220,7 +220,7 @@ def test_builder_cli_fromtag(self):
         self.assertFalse(result.exception)
         self.assertEqual(
             result.output,
-            f"INFO:aswfdocker.builder:Would build: 'docker buildx bake -f {tempfile.gettempdir()}/docker-bake-IMAGE-common-1.json --progress auto'\n",
+            f"INFO:aswfdocker.builder:Would build: 'docker buildx bake -f {tempfile.gettempdir()}/docker-bake-CI_IMAGE-common-1.json --progress auto'\n",
         )
         self.assertEqual(result.exit_code, 0)
 

python/aswfdocker/tests/test_index.py

@@ -22,9 +22,14 @@ def test_iter_images(self):
         self.assertEqual(packages[0], "clang")
 
     def test_get_versions(self):
-        images = list(self.index.iter_images(constants.ImageType.IMAGE))
-        self.assertGreater(len(images), 1)
-        self.assertEqual(images[0], "common")
-        versions = list(self.index.iter_versions(constants.ImageType.IMAGE, images[0]))
+        ciimages = list(self.index.iter_images(constants.ImageType.CI_IMAGE))
+        self.assertGreater(len(ciimages), 1)
+        self.assertEqual(ciimages[0], "common")
+        rtimages = list(self.index.iter_images(constants.ImageType.RT_IMAGE))
+        self.assertGreater(len(rtimages), 1)
+        self.assertEqual(rtimages[0], "base")
+        versions = list(
+            self.index.iter_versions(constants.ImageType.CI_IMAGE, ciimages[0])
+        )
         self.assertGreaterEqual(len(versions), 1)
         self.assertTrue(versions[0].startswith("1."))

python/aswfdocker/tests/test_utils.py

@@ -71,7 +71,11 @@ def test_image_def_from_name(self):
             utils.get_image_spec("aswf/ci-package-openexr_2018")
         self.assertEqual(
             utils.get_image_spec("aswftesting/ci-common:1"),
-            ("aswftesting", constants.ImageType.IMAGE, "common", "1"),
+            ("aswftesting", constants.ImageType.CI_IMAGE, "common", "1"),
+        )
+        self.assertEqual(
+            utils.get_image_spec("aswftesting/rt-base:2019"),
+            ("aswftesting", constants.ImageType.RT_IMAGE, "base", "2019"),
         )
         self.assertEqual(
             utils.get_image_spec("aswf/ci-package-openexr:2018"),
@@ -164,7 +168,7 @@ def test_cli_images(self):
         imgs = result.output.split("\n")
         self.assertGreater(len(imgs), 15)
         common_version = list(
-            index.Index().iter_versions(constants.ImageType.IMAGE, "common")
+            index.Index().iter_versions(constants.ImageType.CI_IMAGE, "common")
         )[0]
         self.assertEqual(
             imgs[0], f"common/ci-common:{common_version}",

python/aswfdocker/utils.py

@@ -82,7 +82,9 @@ def download_package(repo_root: str, docker_org: str, package: str, version: str
 def get_image_name(image_type: constants.ImageType, image: str):
     if image_type == constants.ImageType.PACKAGE:
         return f"ci-package-{image}"
-    return f"ci-{image}"
+    if image_type == constants.ImageType.CI_IMAGE:
+        return f"ci-{image}"
+    return f"rt-{image}"
 
 
 IMAGE_NAME_REGEXC = re.compile(constants.IMAGE_NAME_REGEX)
@@ -97,8 +99,14 @@ def get_image_spec(name: str):
     org = m.group("org")
     if m.group("package"):
         image_type = constants.ImageType.PACKAGE
+    elif m.group("ci"):
+        image_type = constants.ImageType.CI_IMAGE
+    elif m.group("rt"):
+        image_type = constants.ImageType.RT_IMAGE
     else:
-        image_type = constants.ImageType.IMAGE
+        raise RuntimeError(
+            f"Image name does not conform to expected format (missing image type): {constants.IMAGE_NAME_REGEX}"
+        )
     image = m.group("image")
     version = m.group("version")
     logger.debug("get_image_spec found %s: %s/%s:%s", image_type, org, image, version)

rt-base/Dockerfile

@@ -0,0 +1,106 @@
+# Copyright (c) Contributors to the aswf-docker Project. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+ARG CUDA_VERSION=10.2
+ARG ASWF_ORG=aswftesting
+ARG ASWF_PKG_ORG=aswftesting
+ARG CI_COMMON_VERSION=1
+ARG VFXPLATFORM_VERSION=2019
+
+ARG ASWF_ORG
+ARG VFXPLATFORM_VERSION
+ARG ASWF_VERSION
+ARG PYTHON_VERSION=2.7
+
+FROM ${ASWF_PKG_ORG}/ci-package-python:${VFXPLATFORM_VERSION} as ci-package-python
+FROM ${ASWF_PKG_ORG}/ci-package-tbb:${VFXPLATFORM_VERSION} as ci-package-tbb
+
+FROM nvidia/cudagl:${CUDA_VERSION}-devel-centos7 as builder
+
+COPY --from=ci-package-python /. /usr/local/
+COPY --from=ci-package-tbb /. /usr/local/
+
+RUN rm -rf /usr/local/include
+
+FROM nvidia/cudagl:${CUDA_VERSION}-devel-centos7 as rt-base
+
+LABEL maintainer="[email protected]"
+LABEL com.vfxplatform.version=$VFXPLATFORM_VERSION
+LABEL org.opencontainers.image.name="$ASWF_ORG/rt-common"
+LABEL org.opencontainers.image.description="Common image for runtime ASWF images"
+LABEL org.opencontainers.image.url="http://aswf.io/"
+LABEL org.opencontainers.image.source="https://github.com/AcademySoftwareFoundation/aswf-docker"
+LABEL org.opencontainers.image.vendor="AcademySoftwareFoundation"
+LABEL org.opencontainers.image.version=$VFXPLATFORM_VERSION
+
+ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:${LD_LIBRARY_PATH} \
+    PATH=/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \
+    PYTHONPATH=/usr/local/lib/python${PYTHON_VERSION}/site-packages:/usr/local/lib/python \
+    VFXPLATFORM_VERSION=$VFXPLATFORM_VERSION \
+    ASWF_VERSION=${ASWF_VERSION} \
+    ASWF_ORG=${ASWF_ORG}
+
+COPY --from=builder /usr/local /usr/local/
+
+RUN yum install --setopt=tsflags=nodocs -y \
+    alsa-lib \
+    automake autoconf \
+    bison \
+    bzip2 \
+    ca-certificates \
+    csh \
+    cups \
+    dbus \
+    doxygen \
+    fam \
+    file \
+    fontconfig \
+    freeglut \
+    freetype \
+    glx-utils \
+    gstreamer1 \
+    gstreamer1-plugins-bad-free \
+    libgomp \
+    libicu \
+    libjpeg \
+    libpng \
+    libtiff \
+    libv4l \
+    libxcb \
+    libXcomposite \
+    libXcursor \
+    libXi \
+    libXinerama \
+    libxkbcommon \
+    libxkbcommon-x11 \
+    libxml2 \
+    libXmu \
+    libXp \
+    libXpm \
+    libXrandr \
+    libXrender \
+    libXScrnSaver \
+    libxslt \
+    mesa-libGLU \
+    motif \
+    ncurses \
+    nss \
+    patch \
+    pkgconfig \
+    pulseaudio-libs \
+    readline \
+    rsync \
+    ruby \
+    sudo \
+    tcsh \
+    texinfo \
+    unzip \
+    wget \
+    which \
+    xcb-util \
+    xcb-util-image \
+    xcb-util-keysyms \
+    xcb-util-wm \
+    xorg-x11-xkb-utils \
+    xorg-x11-server-Xvfb && \
+    rm -rf /var/cache/yum/*
+