Added RunTime set of images that are slimmer and a jupyter bonus

.github/workflows/docker-builds.yml

@@ -24,12 +24,12 @@ jobs:
         group: [img-common-1] # Extra groups come from the include next
         include:
           - group: img-common-1
-            type: IMAGE
+            type: CI_IMAGE
             name: common
             version: 1
             test: common
           - group: img-common-2
-            type: IMAGE
+            type: CI_IMAGE
             name: common
             version: 2
             test: common
@@ -46,31 +46,31 @@ jobs:
             version: 2021
             test: ""
           - group: img-base-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: base
             common_version: 2
             version: 2021
             test: base
           - group: img-vfx1-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: vfx1
             common_version: 2
             version: 2021
             test: openvdb
           - group: img-vfx2-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: vfx2
             common_version: 2
             version: 2021
             test: ""
           - group: img-usd-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: usd
             common_version: 2
             version: 2021
             test: ""
           - group: img-vfxall-2021
-            type: IMAGE
+            type: CI_IMAGE
             name: vfxall
             common_version: 2
             version: 2021
@@ -116,11 +116,11 @@ jobs:
           --source-branch $GITHUB_REF \
           --verbose \
           build \
-          --ci-image-type IMAGE \
+          --ci-image-type CI_IMAGE \
           --group common \
           --version ${{ matrix.common_version }}
       name: Optionally build local ci-common Docker Image for downstream images
-      if: ${{ matrix.name != 'common' && matrix.type == 'IMAGE' }}
+      if: ${{ matrix.name != 'common' && matrix.type == 'CI_IMAGE' }}
 
     - run: |
         set -ex

CONTRIBUTING.md

@@ -284,7 +284,7 @@ Example use: just build a single package for testing:
 # Build and push USD package to aswftesting
 aswfdocker --verbose build -t PACKAGE --group vfx --version 2019 --target usd --push
 # Build and push ci-vfxall image to aswftesting
-aswfdocker --verbose build -t IMAGE --group vfx --version 2019 --target vfxall --push
+aswfdocker --verbose build -t CI_IMAGE --group vfx --version 2019 --target vfxall --push
 ```
 
 ### Migrate

README.md

@@ -116,15 +116,15 @@ but it is recommended to speed up large builds.
 
 To build all images (very unlikely to succeed unless run on a very very powerful machine!):
 ```bash
-aswfdocker --verbose build -t IMAGE
+aswfdocker --verbose build -t CI_IMAGE
 ```
 
 To build a single image:
 ```bash
 # First list the available CI images to know which package belong to which "group":
 aswfdocker images
 # Then run the build
-aswfdocker --verbose build -t IMAGE --group vfx1 --version 2019 --target openexr
+aswfdocker --verbose build -t CI_IMAGE --group vfx1 --version 2019 --target openexr
 # Or the simpler but less flexible syntax:
 aswfdocker build -n aswftesting/ci-openexr:2019
 ```

python/aswfdocker/cli/aswfdocker.py

@@ -257,15 +257,39 @@ def packages():
 
 @cli.command()
 def images():
-    """Lists all known ci images in this format: IMAGEGROUP/ci-IMAGE:VERSION
+    """Lists all known ci images in this format: IMAGEGROUP/IMAGE:VERSION
     """
-    for group, images in constants.GROUPS[constants.ImageType.IMAGE].items():
-        for image in images:
-            image_name = utils.get_image_name(constants.ImageType.IMAGE, image)
-            for version in index.Index().iter_versions(
-                constants.ImageType.IMAGE, image
-            ):
-                click.echo(f"{group}/{image_name}:{version}")
+    for image_type in (constants.ImageType.CI_IMAGE, constants.ImageType.RT_IMAGE):
+        for group, images in constants.GROUPS[image_type].items():
+            for image in images:
+                image_name = utils.get_image_name(image_type, image)
+                for version in index.Index().iter_versions(image_type, image):
+                    click.echo(f"{group}/{image_name}:{version}")
+
+
+@click.option(
+    "--sizes", "-s", is_flag=True, help="Display Sizes",
+)
+@cli.command()
+def dockerstats(sizes):
+    """Lists all known ci images in this format: IMAGEGROUP/IMAGE:VERSION
+    """
+    if sizes:
+        total_size = 0
+        for org, image_type, image in utils.iter_all_images():
+            image_name = utils.get_image_name(image_type, image)
+            for tag, size in utils.get_image_sizes(org, image_name).items():
+                click.echo(f"{org}/{image_name}:{tag} size={size}")
+                total_size += size
+        click.echo(f"Total size={total_size}")
+    else:
+        total_pull = 0
+        for org, image_type, image in utils.iter_all_images():
+            image_name = utils.get_image_name(image_type, image)
+            pull_count = utils.get_image_pull_count(org, image_name)
+            click.echo(f"{org}/{image_name} pull_count={pull_count}")
+            total_pull += pull_count
+        click.echo(f"Total pull_count={total_pull}")
 
 
 @cli.command()

python/aswfdocker/constants.py

@@ -9,7 +9,8 @@
 
 
 class ImageType(enum.Enum):
-    IMAGE = "image"
+    CI_IMAGE = "ci-image"
+    RT_IMAGE = "rt-image"
     PACKAGE = "package"
 
 
@@ -24,14 +25,19 @@ class ImageType(enum.Enum):
         "vfx2": ["opensubdiv", "ptex", "openvdb", "usd", "otio"],
         "vfx3": ["partio", "osl"],
     },
-    ImageType.IMAGE: {
+    ImageType.CI_IMAGE: {
         "common": ["common"],
         "base": ["base"],
         "vfx1": ["openexr", "openvdb", "opencue"],
         "vfx2": ["ocio", "osl", "otio"],
         "usd": ["usd"],
         "vfxall": ["vfxall"],
     },
+    ImageType.RT_IMAGE: {
+        "base": ["base"],
+        "vfx1": ["vfxall"],
+        "vfx2": ["vfxall-jupyter"],
+    },
 }
 
 VERSION_INFO = {
@@ -95,4 +101,4 @@ class ImageType(enum.Enum):
     f"https://github.com/{MAIN_GITHUB_ASWF_ORG}/{MAIN_GITHUB_REPO_NAME}"
 )
 
-IMAGE_NAME_REGEX = r"(refs/tags/)?(?P<org>[a-z]+)/ci-(?P<package>package\-)?(?P<image>[a-z0-9]+)[:/](?P<version>[0-9\.]+)"
+IMAGE_NAME_REGEX = r"(refs/tags/)?(?P<org>[a-z]+)/(?P<ci>ci-)?(?P<rt>rt-)?(?P<package>package\-)?(?P<image>[a-z0-9\-]+)[:/](?P<version>[0-9\.]+)"

python/aswfdocker/index.py

@@ -24,7 +24,9 @@ def __init__(self):
     def _get_key(self, image_type: constants.ImageType):
         if image_type == constants.ImageType.PACKAGE:
             return "ci-packages"
-        return "ci-images"
+        if image_type == constants.ImageType.CI_IMAGE:
+            return "ci-images"
+        return "rt-images"
 
     def iter_images(self, image_type: constants.ImageType):
         """

python/aswfdocker/tests/test_builder.py

@@ -73,12 +73,12 @@ def test_image_base_2019_dict(self):
             groupinfo.GroupInfo(
                 names=["base"],
                 versions=["2019"],
-                type_=constants.ImageType.IMAGE,
+                type_=constants.ImageType.CI_IMAGE,
                 targets=[],
             ),
         )
         base_version = list(
-            index.Index().iter_versions(constants.ImageType.IMAGE, "base")
+            index.Index().iter_versions(constants.ImageType.CI_IMAGE, "base")
         )[1]
         self.assertEqual(
             b.make_bake_dict(),
@@ -118,12 +118,12 @@ def test_image_base_2019_2020_dict(self):
             groupinfo.GroupInfo(
                 names=["base"],
                 versions=["2019", "2020"],
-                type_=constants.ImageType.IMAGE,
+                type_=constants.ImageType.CI_IMAGE,
                 targets=[],
             ),
         )
         base_versions = list(
-            index.Index().iter_versions(constants.ImageType.IMAGE, "base")
+            index.Index().iter_versions(constants.ImageType.CI_IMAGE, "base")
         )
         self.assertEqual(
             b.make_bake_dict(),
@@ -220,7 +220,7 @@ def test_builder_cli_fromtag(self):
         self.assertFalse(result.exception)
         self.assertEqual(
             result.output,
-            f"INFO:aswfdocker.builder:Would build: 'docker buildx bake -f {tempfile.gettempdir()}/docker-bake-IMAGE-common-1.json --progress auto'\n",
+            f"INFO:aswfdocker.builder:Would build: 'docker buildx bake -f {tempfile.gettempdir()}/docker-bake-CI_IMAGE-common-1.json --progress auto'\n",
         )
         self.assertEqual(result.exit_code, 0)
 

python/aswfdocker/tests/test_index.py

@@ -22,9 +22,14 @@ def test_iter_images(self):
         self.assertEqual(packages[0], "clang")
 
     def test_get_versions(self):
-        images = list(self.index.iter_images(constants.ImageType.IMAGE))
-        self.assertGreater(len(images), 1)
-        self.assertEqual(images[0], "common")
-        versions = list(self.index.iter_versions(constants.ImageType.IMAGE, images[0]))
+        ciimages = list(self.index.iter_images(constants.ImageType.CI_IMAGE))
+        self.assertGreater(len(ciimages), 1)
+        self.assertEqual(ciimages[0], "common")
+        rtimages = list(self.index.iter_images(constants.ImageType.RT_IMAGE))
+        self.assertGreater(len(rtimages), 1)
+        self.assertEqual(rtimages[0], "base")
+        versions = list(
+            self.index.iter_versions(constants.ImageType.CI_IMAGE, ciimages[0])
+        )
         self.assertGreaterEqual(len(versions), 1)
         self.assertTrue(versions[0].startswith("1."))

python/aswfdocker/tests/test_utils.py

@@ -71,7 +71,11 @@ def test_image_def_from_name(self):
             utils.get_image_spec("aswf/ci-package-openexr_2018")
         self.assertEqual(
             utils.get_image_spec("aswftesting/ci-common:1"),
-            ("aswftesting", constants.ImageType.IMAGE, "common", "1"),
+            ("aswftesting", constants.ImageType.CI_IMAGE, "common", "1"),
+        )
+        self.assertEqual(
+            utils.get_image_spec("aswftesting/rt-base:2019"),
+            ("aswftesting", constants.ImageType.RT_IMAGE, "base", "2019"),
         )
         self.assertEqual(
             utils.get_image_spec("aswf/ci-package-openexr:2018"),
@@ -164,7 +168,7 @@ def test_cli_images(self):
         imgs = result.output.split("\n")
         self.assertGreater(len(imgs), 15)
         common_version = list(
-            index.Index().iter_versions(constants.ImageType.IMAGE, "common")
+            index.Index().iter_versions(constants.ImageType.CI_IMAGE, "common")
         )[0]
         self.assertEqual(
             imgs[0], f"common/ci-common:{common_version}",

python/aswfdocker/utils.py

@@ -8,6 +8,8 @@
 import subprocess
 import datetime
 import logging
+import json
+import urllib.request
 
 from aswfdocker import constants
 
@@ -82,7 +84,9 @@ def download_package(repo_root: str, docker_org: str, package: str, version: str
 def get_image_name(image_type: constants.ImageType, image: str):
     if image_type == constants.ImageType.PACKAGE:
         return f"ci-package-{image}"
-    return f"ci-{image}"
+    if image_type == constants.ImageType.CI_IMAGE:
+        return f"ci-{image}"
+    return f"rt-{image}"
 
 
 IMAGE_NAME_REGEXC = re.compile(constants.IMAGE_NAME_REGEX)
@@ -97,8 +101,14 @@ def get_image_spec(name: str):
     org = m.group("org")
     if m.group("package"):
         image_type = constants.ImageType.PACKAGE
+    elif m.group("ci"):
+        image_type = constants.ImageType.CI_IMAGE
+    elif m.group("rt"):
+        image_type = constants.ImageType.RT_IMAGE
     else:
-        image_type = constants.ImageType.IMAGE
+        raise RuntimeError(
+            f"Image name does not conform to expected format (missing image type): {constants.IMAGE_NAME_REGEX}"
+        )
     image = m.group("image")
     version = m.group("version")
     logger.debug("get_image_spec found %s: %s/%s:%s", image_type, org, image, version)
@@ -111,3 +121,43 @@ def get_group_from_image(image_type: constants.ImageType, image: str):
             if img == image:
                 return group
     raise RuntimeError(f"Cannot find group for image {image}")
+
+
+def get_image_pull_count(docker_org, image):
+    url = f"https://hub.docker.com/v2/repositories/{docker_org}/{image}"
+    try:
+        d = json.loads(urllib.request.urlopen(url).read())
+        return d["pull_count"]
+    except urllib.error.HTTPError:
+        logger.debug("Failed to load data from URL %r", url)
+        return 0
+
+
+def get_image_sizes(docker_org, image):
+    sizes = {}
+    url = f"https://hub.docker.com/v2/repositories/{docker_org}/{image}/tags/"
+    try:
+        d = json.loads(urllib.request.urlopen(url).read())
+    except urllib.error.HTTPError:
+        logger.debug("Failed to load data from URL %r", url)
+        return sizes
+    digests = set()
+    for tag in d["results"]:
+        digest = tag["images"][0]["digest"]
+        if digest in digests:
+            continue
+        digests.add(digest)
+        sizes[tag["name"]] = tag["full_size"]
+    return sizes
+
+
+def iter_all_images():
+    for org in (constants.TESTING_DOCKER_ORG, constants.PUBLISH_DOCKER_ORG):
+        for image_type in (
+            constants.ImageType.PACKAGE,
+            constants.ImageType.CI_IMAGE,
+            constants.ImageType.RT_IMAGE,
+        ):
+            for _, images in constants.GROUPS[image_type].items():
+                for image in images:
+                    yield org, image_type, image