.nvidia-ci.yml

# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

include:
    - local: '.common-ci.yml'

default:
  tags:
    - cnt
    - container-dev
    - docker/multi-arch
    - docker/privileged
    - os/linux
    - type/docker

variables:
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: "/certs"
  # Release "devel"-tagged images off the master branch
  # Disabling this for now
  RELEASE_DEVEL_BRANCH: "DISABLED"
  DEVEL_RELEASE_IMAGE_VERSION: "devel"
  # Define the public staging registry
  STAGING_REGISTRY: ghcr.io/nvidia
  STAGING_VERSION: "${CI_COMMIT_SHORT_SHA}"

.image-pull-rules:
  # We delay the job start to allow the public pipeline to generate the required images.
  rules:
    - when: delayed
      start_in: 30 minutes

.image-pull-generic:
  stage: image-build
  variables:
    IN_REGISTRY: "${STAGING_REGISTRY}"
    IN_IMAGE_NAME: driver
    IN_VERSION: "${STAGING_VERSION}"
    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
    OUT_REGISTRY: "${CI_REGISTRY}"
    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}"
  timeout: 30 minutes
  retry:
    max: 2
    when:
      - job_execution_timeout
      - stuck_or_timeout_failure
  before_script:
    - !reference [.regctl-setup, before_script]
    - 'if [[ "${PRECOMPILED}" != "true" ]]; then echo DRIVER_VERSION: ${DRIVER_VERSION} ; [[ -n "${DRIVER_VERSION}" ]] || exit 1; fi'
    - apk add --no-cache make bash
    #- regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DRIVER_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DRIVER_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DRIVER_VERSION}-${DIST} does not exist" && sleep infinity )
  script:
    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
    - if [[ "${PRECOMPILED}" == "true" ]]; then IMAGE_NAME=${IN_REGISTRY}/${IN_IMAGE_NAME} ./scripts/precompiled.sh push; else make IMAGE_NAME=${IN_REGISTRY}/${IN_IMAGE_NAME} VERSION=${IN_VERSION} OUT_VERSION=${IN_VERSION} push-${DIST}-${DRIVER_VERSION}; fi

.image-pull:
  # Perform for each DRIVER_VERSION
  extends:
    - .driver-versions
    - .image-pull-generic
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - !reference [.image-pull-rules, rules]


.image-pull-ubuntu24.04:
  # Perform for each DRIVER_VERSION
  extends:
    - .driver-versions
    - .image-pull-generic
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - !reference [.image-pull-rules, rules]

image-precompiled-ubuntu24.04:
  variables:
    DIST: signed_ubuntu24.04
    BASE_TARGET: noble
    PRECOMPILED: "true"
    CVE_UPDATES: "curl libc6"
  rules:
    - when: delayed
      start_in: 30 minutes
  extends:
    - .driver-versions-precompiled-ubuntu24.04
    - .image-pull-generic


.image-pull-ubuntu22.04:
  # Perform for each DRIVER_VERSION
  extends:
    - .driver-versions
    - .image-pull-generic
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - !reference [.image-pull-rules, rules]

image-precompiled-ubuntu22.04:
  variables:
    DIST: signed_ubuntu22.04
    BASE_TARGET: jammy
    PRECOMPILED: "true"
    CVE_UPDATES: "curl libc6"
  rules:
    - when: delayed
      start_in: 30 minutes
  extends:
    - .driver-versions-precompiled-ubuntu22.04
    - .image-pull-generic

image-ubuntu20.04:
  extends:
    - .image-pull
    - .dist-ubuntu20.04

image-ubuntu22.04:
  extends:
    - .image-pull-ubuntu22.04
    - .dist-ubuntu22.04

image-rhel8:
  extends:
    - .image-pull
    - .dist-rhel8

# The .scan step forms the base of the image scan operation performed before releasing
# images.
.scan-generic:
  stage: scan
  image: "${PULSE_IMAGE}"
  variables:
    IMAGE_NAME: "${CI_REGISTRY_IMAGE}"
    VERSION: "${STAGING_VERSION}"
  rules:
    - if: $CI_COMMIT_MESSAGE =~ /\[skip[ _-]scans?\]/i
      when: never
    - if: $SKIP_SCANS && $SKIP_SCANS == "yes"
      when: never
  before_script:
    - !reference [.regctl-setup, before_script]
    # We ensure that the DRIVER_VERSION and PLATFORM are set
    - 'if [[ "${PRECOMPILED}" != "true" ]]; then echo DRIVER_VERSION: ${DRIVER_VERSION} ; [[ -n "${DRIVER_VERSION}" ]] || exit 1; fi'
    - 'echo PLATFORM: ${PLATFORM} ; [[ -n "${PLATFORM}" ]] || exit 1'
    - apk add --no-cache make bash
    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
    - if [[ "${PRECOMPILED}" == "true" ]]; then ./scripts/precompiled.sh pull; else make pull-${DIST}-${DRIVER_VERSION}; fi
    - if [[ "${PRECOMPILED}" == "true" ]]; then ./scripts/precompiled.sh archive; else make archive-${DIST}-${DRIVER_VERSION}; fi
    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
    - >
      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
  script:
    - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i "archive.tar" -p $CONTAINER_POLICY -o
  artifacts:
    when: always
    expire_in: 1 week
    paths:
      - pulse-cli.log
      - licenses.json
      - sbom.json
      - vulns.json
      - policy_evaluation.json

.scan-rules-common:
  rules:
    - if: $CI_COMMIT_MESSAGE =~ /\[skip[ _-]scans?\]/i
      when: never
    - if: $SKIP_SCANS && $SKIP_SCANS == "yes"
      when: never

.scan:
  # Repeat for each DRIVER_VERSION
  extends:
    - .driver-versions
    - .scan-generic
  rules:
    - !reference [.scan-rules-common, rules]
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - !reference [.pipeline-trigger-rules, rules]

.scan-ubuntu22.04:
  # Repeat for each DRIVER_VERSION
  extends:
    - .driver-versions
    - .scan-generic
  rules:
    - !reference [.scan-rules-common, rules]
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - !reference [.pipeline-trigger-rules, rules]

.scan-ubuntu24.04:
  # Repeat for each DRIVER_VERSION
  extends:
    - .driver-versions-ubuntu24.04
    - .scan-generic
  rules:
    - !reference [.scan-rules-common, rules]
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - !reference [.pipeline-trigger-rules, rules]

.scan-precompiled-ubuntu24.04:
  variables:
    DIST: signed_ubuntu24.04
    BASE_TARGET: noble
    PRECOMPILED: "true"
  extends:
    - .driver-versions-precompiled-ubuntu24.04
    - .scan-generic
  rules:
    - !reference [.scan-rules-common, rules]
    - when: always

.scan-precompiled-ubuntu22.04:
  variables:
    DIST: signed_ubuntu22.04
    BASE_TARGET: jammy
    PRECOMPILED: "true"
  extends:
    - .driver-versions-precompiled-ubuntu22.04
    - .scan-generic
  rules:
    - !reference [.scan-rules-common, rules]
    - when: always

# Define the scan targets
scan-ubuntu20.04-amd64:
  extends:
    - .scan
    - .dist-ubuntu20.04
    - .platform-amd64
  needs:
    - image-ubuntu20.04

scan-ubuntu20.04-arm64:
  extends:
    - .scan
    - .dist-ubuntu20.04
    - .platform-arm64
  needs:
    - image-ubuntu20.04

scan-ubuntu22.04-amd64:
  extends:
    - .scan-ubuntu22.04
    - .dist-ubuntu22.04
    - .platform-amd64
  needs:
    - image-ubuntu22.04

scan-ubuntu22.04-arm64:
  extends:
    - .scan-ubuntu22.04
    - .dist-ubuntu22.04
    - .platform-arm64
  needs:
    - image-ubuntu22.04

scan-ubuntu24.04-amd64:
  extends:
    - .scan-ubuntu24.04
    - .dist-ubuntu24.04
    - .platform-amd64
  needs:
    - image-ubuntu24.04

scan-ubuntu24.04-arm64:
  extends:
    - .scan-ubuntu24.04
    - .dist-ubuntu24.04
    - .platform-arm64
  needs:
    - image-ubuntu24.04

scan-precompiled-ubuntu22.04-amd64:
  variables:
    PLATFORM: linux/amd64
  extends:
    - .scan-precompiled-ubuntu22.04
    - .platform-amd64
  needs:
    - image-precompiled-ubuntu22.04

scan-rhel8-amd64:
  extends:
    - .scan
    - .dist-rhel8
    - .platform-amd64
  needs:
    - image-rhel8

scan-rhel8-arm64:
  extends:
    - .scan
    - .dist-rhel8
    - .platform-arm64
  needs:
    - image-rhel8

.release:ngc-variables:
  variables:
    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
    OUT_REGISTRY: "${NGC_REGISTRY}"
    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"

# Define the external release helpers
.release:ngc:
  extends:
    - .release:external
    - .release:ngc-variables

release:ngc-ubuntu20.04:
  extends:
    - .release:ngc
    - .dist-ubuntu20.04

release:ngc-ubuntu22.04:
  extends:
    - .release:ngc
    - .dist-ubuntu22.04
    - .driver-versions

# TODO will be enabled after QA
# release:ngc-ubuntu24.04:
#   extends:
#     - .release:ngc
#     - .dist-ubuntu24.04
#     - .driver-versions

# release:ngc-precompiled-ubuntu24.04:
#   variables:
#     DIST: signed_ubuntu24.04
#     BASE_TARGET: noble
#     PRECOMPILED: "true"
#   extends:
#     - .driver-versions-precompiled-ubuntu24.04
#     - .release-generic
#     - .release:ngc-variables
#   rules:
#     # Only run NGC release job on scheduled pipelines
#     - if: $CI_PIPELINE_SOURCE == "schedule"

release:ngc-precompiled-ubuntu22.04:
  variables:
    DIST: signed_ubuntu22.04
    BASE_TARGET: jammy
    PRECOMPILED: "true"
  extends:
    - .driver-versions-precompiled-ubuntu22.04
    - .release-generic
    - .release:ngc-variables
  rules:
    # Only run NGC release job on scheduled pipelines
    - if: $CI_PIPELINE_SOURCE == "schedule"

release:ngc-rhcos4.12:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhcos4.12"

release:ngc-rhcos4.13:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhcos4.13"

release:ngc-rhcos4.14:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhcos4.14"

release:ngc-rhcos4.15:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhcos4.15"

release:ngc-rhcos4.16:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhcos4.16"

release:ngc-rhcos4.17:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhcos4.17"

release:ngc-rhel8.8:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhel8.8"

release:ngc-rhel8.10:
  extends:
    - .release:ngc
    - .dist-rhel8
  variables:
    OUT_DIST: "rhel8.10"

# Define the external image signing steps for NGC
# Download the ngc cli binary for use in the sign steps
.ngccli-setup:
  before_script:
    - apt-get update && apt-get install -y curl unzip jq bash make
    - export REGCTL_VERSION=v0.7.1
    - mkdir -p bin
    - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
    - chmod a+x bin/regctl
    - export PATH=$(pwd)/bin:${PATH}
    - |
      if [ -z "${NGCCLI_VERSION}" ]; then
        NGC_VERSION_URL="https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions"
        # Extract the latest version from the JSON data using jq
        export NGCCLI_VERSION=$(curl -s $NGC_VERSION_URL | jq -r '.recipe.latestVersionIdStr')
      fi
      echo "NGCCLI_VERSION ${NGCCLI_VERSION}"
    - curl -sSLo ngccli_linux.zip https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGCCLI_VERSION}/files/ngccli_linux.zip
    - unzip ngccli_linux.zip
    - chmod u+x ngc-cli/ngc

# .sign forms the base of the deployment jobs which signs images in the CI registry.
# This is extended with the image name and version to be deployed.
.sign:ngc:
  image: ubuntu:latest
  timeout: 30 minutes
  when: always
  stage: sign
  rules:
    - if: $CI_COMMIT_TAG
      variables:
        DIST: "${SIGN_JOB_NAME}${VERSION}"
  variables:
    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
    OUT_REGISTRY: "${CI_REGISTRY}"
    NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
    IMAGE_TAG: "${CI_COMMIT_TAG}"
  before_script:
    - !reference [.ngccli-setup, before_script]
    - 'if [[ "${PRECOMPILED}" != "true" ]]; then echo DRIVER_VERSION: ${DRIVER_VERSION} ; [[ -n "${DRIVER_VERSION}" ]] || exit 1; fi'
    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
    - |
      if [[ "${PRECOMPILED}" == "true" ]]; then
        ./scripts/precompiled.sh version;
        source kernel_version.txt
        export IMAGE_TAG=${DRIVER_BRANCH}-${KERNEL_VERSION}-${DIST##*_}
      else
          export IMAGE_TAG=${DRIVER_VERSION}-${DIST##*_}
      fi
    # We ensure that the IMAGE_NAME and IMAGE_TAG is set
    - 'echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1'
    - 'echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1'
  script:
    - 'echo "Signing the image ${IMAGE_NAME}:${IMAGE_TAG}"'
    - ngc-cli/ngc registry image publish --source ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${IMAGE_TAG} --public --discoverable --allow-guest --sign --org nvidia

#sign:ngc-precompiled-ubuntu24.04:
#  extends:
#    - .driver-versions-precompiled-ubuntu24.04
#    - .dist-ubuntu22.04
#    - .release-generic
#    - .release:ngc-variables
#    - .sign:ngc
#  variables:
#    DIST: signed_ubuntu24.04
#    BASE_TARGET: noble
#    PRECOMPILED: "true"
#  needs:
#    - release:ngc-precompiled-ubuntu24.04
#  rules:
#    # Only run NGC release job on scheduled pipelines
#    - if: $CI_PIPELINE_SOURCE == "schedule"

sign:ngc-precompiled-ubuntu22.04:
  extends:
    - .driver-versions-precompiled-ubuntu22.04
    - .dist-ubuntu22.04
    - .release-generic
    - .release:ngc-variables
    - .sign:ngc
  variables:
    DIST: signed_ubuntu22.04
    BASE_TARGET: jammy
    PRECOMPILED: "true"
  needs:
    - release:ngc-precompiled-ubuntu22.04
  rules:
    # Only run NGC release job on scheduled pipelines
    - if: $CI_PIPELINE_SOURCE == "schedule"

sign:ngc-ubuntu-rhel-rhcos:
  extends:
    - .sign:ngc
  parallel:
    matrix:
#    - SIGN_JOB_NAME: ["ubuntu"]
#      VERSION: ["24.04"]
#      DRIVER_VERSION: ["550.127.08"]
    - SIGN_JOB_NAME: ["ubuntu"]
      VERSION: ["22.04"]
      DRIVER_VERSION: ["535.216.03", "550.127.08"]
    - SIGN_JOB_NAME: ["ubuntu"]
      VERSION: ["20.04"]
      DRIVER_VERSION: ["535.216.03", "550.127.08"]
    - SIGN_JOB_NAME: ["rhel"]
      VERSION: ["8.8", "8.10"]
      DRIVER_VERSION: ["535.216.03", "550.127.08"]
    - SIGN_JOB_NAME: ["rhcos"]
      VERSION: ["4.12","4.13","4.14","4.15", "4.16", "4.17"]
      DRIVER_VERSION: ["535.216.03", "550.127.08"]