Merge branch 'cloudera-labs:devel' into devel

.github/workflows/build_push_image.yml

@@ -31,7 +31,7 @@ on:
         value: ${{ jobs.build-push.outputs.registry-paths }}
       image-sha:
         description: 'the SHA-tagged image path'
-        value: ghcr.io/${{ github.repository }}:sha-${{ jobs.build-push.outputs.sha-short }}
+        value: ghcr.io/${{ github.repository }}-${{ inputs.PROFILE }}:sha-${{ jobs.build-push.outputs.sha-short }}
 
 permissions:
   contents: read
@@ -45,26 +45,25 @@ jobs:
       registry-paths: ${{ steps.push-image.outputs.registry-paths }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install latest ansible-builder
         run: pip install ansible-builder
 
       - name: Construct image metadata
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: |
-            ghcr.io/${{ github.repository }}
+            ghcr.io/${{ github.repository }}-${{ inputs.PROFILE }}
           flavor: |
             latest=auto
-            prefix=${{ inputs.PROFILE }}-,onlatest=true
           tags: |
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=ref,event=branch
             type=ref,event=pr
-            type=sha,prefix=${{ inputs.PROFILE }}-
+            type=sha
 
       - name: Construct short SHA
         id: sha
@@ -73,14 +72,14 @@ jobs:
       - name: Update container base image reference
         if: ${{ inputs.PROFILE != 'base' }}
         env:
-          REPLACEMENT_IMAGE: ${{ github.repository }}:base-${{ steps.sha.outputs.sha-short }}
+          REPLACEMENT_IMAGE: ${{ github.repository }}-base:sha-${{ steps.sha.outputs.sha-short }}
         run:
-          sed -i 's|cloudera-labs/cldr-runner:base-latest|${{ env.REPLACEMENT_IMAGE }}|' ${{ inputs.PROFILE }}/execution-environment.yml
+          sed -i 's|cloudera-labs/cldr-runner-base:latest|${{ env.REPLACEMENT_IMAGE }}|' ${{ inputs.PROFILE }}/execution-environment.yml
 
       - name: Create builder context
         run: |
           ansible-builder create --file ${{ inputs.PROFILE }}/execution-environment.yml
-      
+
       - name: Build image from builder context
         uses: redhat-actions/buildah-build@v2
         with:

.github/workflows/label_pr.yml

@@ -66,4 +66,4 @@ jobs:
         uses: actions-ecosystem/action-add-labels@v1
         with:
           labels: validated
-          number: ${{ steps.read.outputs.pr_number }}
+          number: ${{ steps.read.outputs.pr_number }}

.github/workflows/pre-commit.yml

@@ -0,0 +1,30 @@
+# Copyright 2024 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Execute Precommit Linting and Checks
+
+on:
+  pull_request:
+  push:
+    branches: [main, devel]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - uses: pre-commit/[email protected]

.github/workflows/publish_single_image.yml

@@ -46,4 +46,4 @@ jobs:
     needs: publish-base
     uses: ./.github/workflows/build_push_image.yml
     with:
-      PROFILE: ${{ inputs.PROFILE }}
+      PROFILE: ${{ inputs.PROFILE }}

.github/workflows/reset_pr.yml

@@ -37,4 +37,4 @@ jobs:
       - name: Reset the PR label
         uses: actions-ecosystem/action-remove-labels@v1
         with:
-          labels: validated
+          labels: validated

.github/workflows/validate_image.yml

@@ -35,26 +35,25 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install latest ansible-builder
         run: pip install ansible-builder
-    
+
       - name: Construct image metadata
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: |
-            ghcr.io/${{ github.repository }}
+            ghcr.io/${{ github.repository }}-${{ inputs.PROFILE }}
           flavor: |
             latest=auto
-            prefix=${{ inputs.PROFILE }}-,onlatest=true
           tags: |
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=ref,event=branch
             type=ref,event=pr
-            type=sha,prefix=${{ inputs.PROFILE }}-
+            type=sha
 
       - name: Construct short SHA
         id: sha
@@ -63,23 +62,23 @@ jobs:
       - name: Update container base image reference
         if: ${{ inputs.PROFILE != 'base' }}
         env:
-          REPLACEMENT_IMAGE: ${{ github.repository }}:base-${{ steps.sha.outputs.sha-short }}
+          REPLACEMENT_IMAGE: ${{ github.repository }}-base:sha-${{ steps.sha.outputs.sha-short }}
         run:
-          sed -i 's|cloudera-labs/cldr-runner:base-latest|${{ env.REPLACEMENT_IMAGE }}|' ${{ inputs.PROFILE }}/execution-environment.yml
+          sed -i 's|cloudera-labs/cldr-runner-base:latest|${{ env.REPLACEMENT_IMAGE }}|' ${{ inputs.PROFILE }}/execution-environment.yml
 
       - name: Create builder context
         run: |
           ansible-builder create --file ${{ inputs.PROFILE }}/execution-environment.yml
 
       - name: Upload Containerfile
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.PROFILE }}-Containerfile
           path: ./context/Containerfile
-      
+
       - name: Download the base image
         if: ${{ inputs.PROFILE != 'base' }}
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: base-${{ steps.sha.outputs.sha-short }}
           path: /tmp
@@ -106,13 +105,13 @@ jobs:
       - name: Save the base image
         if: ${{ inputs.PROFILE == 'base' }}
         env:
-          BASE_IMAGE: base-${{ steps.sha.outputs.sha-short }}
+          BASE_IMAGE: ${{ steps.sha.outputs.sha-short }}
         run: |
-          podman save --output /tmp/${{ env.BASE_IMAGE }}.tar ${{ github.repository }}:${{ env.BASE_IMAGE }}
+          podman save --output /tmp/base-${{ env.BASE_IMAGE }}.tar ${{ github.repository }}-base:sha-${{ env.BASE_IMAGE }}
 
       - name: Upload base image
         if: ${{ inputs.PROFILE == 'base' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: base-${{ steps.sha.outputs.sha-short }}
           path: /tmp/base-${{ steps.sha.outputs.sha-short }}.tar

.github/workflows/validate_pr.yml

@@ -22,7 +22,7 @@ on:
       - 'release/**'
       - 'devel'
   workflow_dispatch:
-    
+
 jobs:
 
   validate-base:
@@ -51,9 +51,9 @@ jobs:
         run: |
           mkdir -p ./pr
           echo $PR_NUMBER > ./pr/pr_number
-      
+
       - name: Upload the PR number
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pr_number
           path: pr/

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+# Copyright 2024 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black

CONTRIBUTING.md

@@ -4,7 +4,7 @@ Thank you for considering contributions to the `cldr-runner` project!
 
 # Submitting a pull request
 
-You can start work on issues that are not yet part of a [Milestone](https://github.com/cloudera-labs/cldr-runner/milestones) -- anything in our issue tracker that isn't assigned to a Milestone is considered the [backlog](https://github.com/cloudera-labs/cldr-runner/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone). 
+You can start work on issues that are not yet part of a [Milestone](https://github.com/cloudera-labs/cldr-runner/milestones) -- anything in our issue tracker that isn't assigned to a Milestone is considered the [backlog](https://github.com/cloudera-labs/cldr-runner/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone).
 
 Before you start working, please announce that you want to do so by commenting on the issue. _([Create an issue](https://github.com/cloudera-labs/cldr-runner/issues/new?labels=enhancement) if there isn't one yet, and you can also check out our [Discussions](https://github.com/cloudera-labs/cldr-runner/discussions) for ideas.)_ We try to ensure that all active work is assigned to a Milestone in order to keep our backlog accurate.
 
@@ -21,6 +21,16 @@ As a _best practice_, you can prefix your branches with:
 > [!NOTE]
 > :fire_extinguisher: A **hotfix** should branch from `main`. It will then be committed to both the `main` and `devel` branches.
 
+## Running pre-commit
+
+We use [pre-commit](https://pre-commit.com/) to lint and validate submissions.
+
+Setting up `pre-commit` is straightforward; see the [Installation](https://pre-commit.com/#installation) documentation on the `pre-commit` site for details.
+
+We highly suggest you install the local `git` hook scripts as well; run `pre-commit install` within the root of the project.
+
+Lastly, you can run the hooks prior to committing changes (and PR submission) by executing `pre-commit run -a`.
+
 # Signing your commits
 
 Note that we require signed commits inline with [Developer Certificate of Origin](https://developercertificate.org/) best-practices for open source collaboration.

FAQ.md

@@ -1,18 +1,18 @@
 # Frequently Asked Questions
 
-Be sure to check out the [Discussions > Help](https://github.com/cloudera-labs/cldr-runner/discussions/categories/help) category for the latest answers. 
+Be sure to check out the [Discussions > Help](https://github.com/cloudera-labs/cldr-runner/discussions/categories/help) category for the latest answers.
 
 # `ansible-navigator` FAQ
 
 ## How do I add _extra variables_ and tags to `ansible-navigator`?
 
-If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. 
+If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`.
 
 Like tags, so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`.
 
 ## How do I tell `ansible-navigator` where to find collections and roles?
 
-By default, `cloudera-deploy` expects to use the collections, roles, and libraries within the _execution environment_ container, that is, the `cldr-runner` image. Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and pass them to the running container. The underlying `ansible` application, like `ansible-playbook` will then pick up these environment variables and attempt to use them if set! 
+By default, `cloudera-deploy` expects to use the collections, roles, and libraries within the _execution environment_ container, that is, the `cldr-runner` image. Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and pass them to the running container. The underlying `ansible` application, like `ansible-playbook` will then pick up these environment variables and attempt to use them if set!
 
 This behavior is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. See [Advanced Usage: Execution Modes](NAVIGATOR.md#advanced-usage-execution-modes) to learn more about these execution modes.
 

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Cloudera, Inc.
+   Copyright 2024 Cloudera, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

NAVIGATOR.md

@@ -86,7 +86,7 @@ Typically, this collections directory is a mix of Git-cloned and downloaded coll
 
 See the [Placement of Ansible collections](https://ansible.readthedocs.io/projects/navigator/faq/#placement-of-ansible-collections) section for further details.
 
-The [local development](README.md#local-development) instructions are designed to support this mode. 
+The [local development](README.md#local-development) instructions are designed to support this mode.
 
 ## Platform Mode
 

README.md

@@ -6,7 +6,7 @@
 
 Specifically, the project consists of `execution-environment.yml` configuration files and other supporting assets that power [`ansible-builder`](https://ansible.readthedocs.io/projects/builder/en/latest/). The configurations encapsulate the necessary Ansible collections and roles, Python libraries, and system applications to work with Cloudera's products and cloud providers. Moreover, the resulting images have the needed tooling for managing infrastructure if so requested.
 
-`cldr-runner` builds several variations:
+`cldr-runner` builds several profiles:
 
 | Tag | Description |
 |-----|-------------|
@@ -16,6 +16,8 @@ Specifically, the project consists of `execution-environment.yml` configuration
 | [gcp](gcp/execution-environment.yml) | `base` plus GCP-specific collections and dependencies, including the `gcloud` CLI |
 | [full](full/execution-environment.yml) | All of the above, plus additional CLI tools for in-container usage, e.g. `git`, `vim`, `nano`, `tree`, `kubectl` |
 
+Each image is tagged `cloudera-labs/cldr-runner-<profile>:<version>`.
+
 # Quickstart
 
 `cldr-runner` is designed to run with `ansible-navigator` and other _Execution Environment_-based tools. You might want to [install `ansible-navigator`](NAVIGATOR.md) before delving deeper.
@@ -43,7 +45,7 @@ For more information on how to get involved with the `cldr-runner` project, head
 
 You can run Ansible within `cldr-runner` Execution Environments a couple of different ways. Here are the most common:
 
-## `ansible-navigator` 
+## `ansible-navigator`
 
 Using a `cldr-runner` image in the [`ansible-navigator` application](https://ansible.readthedocs.io/projects/navigator/) as the designated [Execution Environment](https://docs.ansible.com/ansible/devel/getting_started_ee/index.html) is straightforward. Update your `ansible-navigator.yml` configuration file to enable the image:
 
@@ -52,15 +54,15 @@ ansible-navigator:
   execution-environment:
     container-engine: docker
     enabled: True
-    image: ghcr.io/cloudera-labs/cldr-runner:aws-latest
+    image: ghcr.io/cloudera-labs/cldr-runner-aws:latest
     pull:
       policy: missing
 ```
 
 Once defined, you can run your Ansible activities within the resulting `cldr-runner` container, e.g. `ansible-navigator run your_playbook.yml`. (You can specify the image via the `ansible-navigator` CLI; set `--eei` or `--execution-environment-image`.)
 
 > [!NOTE]
-> If you want to "drop into" the container directly, i.e. run a shell within the container, run `ansible-navigator exec -- /bin/bash` and all the mounts, environment variables, etc. are handled for you!! Now from the shell, you can still run `ansible-playbook` and all other Ansible applications. 
+> If you want to "drop into" the container directly, i.e. run a shell within the container, run `ansible-navigator exec -- /bin/bash` and all the mounts, environment variables, etc. are handled for you!! Now from the shell, you can still run `ansible-playbook` and all other Ansible applications.
 
 ## AWX/AAP
 
@@ -78,7 +80,7 @@ Once defined, the EE can be used by Job Templates, Container Groups, etc.
 You can run the container directly in `docker` (or `podman`):
 
 ```bash
-docker run -it ghcr.io/cloudera-labs/cldr-runner:aws-latest /bin/bash
+docker run -it ghcr.io/cloudera-labs/cldr-runner-aws:latest /bin/bash
 ```
 
 Take care to assemble and mount the needed directories other supporting assets; the image is based on [`ansible-runner`](https://ansible.readthedocs.io/projects/runner/en/stable/) (as are all Execution Environments) and runs as such.
@@ -97,7 +99,7 @@ python -m venv ~/location/of/venv; source ~/location/of/venv/bin/activate; pip i
 Then change into the directory of the `cldr-runner` variation you need to build and run:
 
 ```bash
-ansible-builder build --prune-images --squash all --build-arg BUILD_VER=<your version> --build-arg BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") --tag <your tag> 
+ansible-builder build --prune-images --squash all --build-arg BUILD_VER=<your version> --build-arg BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") --tag <your tag>
 ```
 
 You may want to update the variation's `execution-environment.yml` configuration file to use a different base image, say a local image, or build the `base` image before constructing CSP or `full` image. You can make this change in the following section of the configuration file:
@@ -121,7 +123,7 @@ version: 3
 
 images:
   base_image:
-    name: ghcr.io/cloudera-labs/cldr-runner:aws-latest
+    name: ghcr.io/cloudera-labs/cldr-runner-aws:latest
 
 dependencies:
   galaxy:
@@ -160,7 +162,7 @@ ansible-navigator:
 
 # Local Development
 
-The `cldr-runner` project can also be used to bootstrap a local development environment on the native host environment (as opposed to an Execution Environment image).  This option is more involved, but can avoid issues with Docker, such as mount latencies and SSH agent forwarding, and improve overall Ansible collection development. 
+The `cldr-runner` project can also be used to bootstrap a local development environment on the native host environment (as opposed to an Execution Environment image).  This option is more involved, but can avoid issues with Docker, such as mount latencies and SSH agent forwarding, and improve overall Ansible collection development.
 
 The `local_development.yml` playbook sets up a `cldr-runner`-like workspace for OSX and Ubuntu.  The playbook will clone the Cloudera collections and `cdpy` for local work, install the external Ansible dependencies, update the Python `venv`, and create a convenient setup script for future work.
 
@@ -209,7 +211,7 @@ Follow these steps to set up a local environment:
 
 # License and Copyright
 
-Copyright 2023, Cloudera, Inc.
+Copyright 2024, Cloudera, Inc.
 
 ```
 Licensed under the Apache License, Version 2.0 (the "License");