values-baremetal-hub.yaml

# Bare metal hub for multi-cluster confidential containers.
# Deploys Trustee (KBS + attestation), ACM, Vault, and full baremetal infrastructure.
# Manages spoke clusters imported via ACM with the baremetal-spoke clusterGroup.
# Trustee is configured for both Intel TDX and AMD SEV-SNP attestation simultaneously.
# Set main.clusterGroupName: baremetal-hub in values-global.yaml to use.

clusterGroup:
  name: baremetal-hub
  isHubCluster: true
  namespaces:
  - open-cluster-management
  - vault
  - external-secrets-operator:
      operatorGroup: true
      targetNamespaces: []
  - external-secrets
  - openshift-sandboxed-containers-operator
  - trustee-operator-system
  - cert-manager-operator
  - cert-manager
  - hello-openshift
  - kbs-access
  - openshift-cnv
  - openshift-storage
  - openshift-nfd
  - baremetal
  - intel-dcap
  - intel-dcap-operator-system:
      operatorGroup: true
      targetNamespaces: []
  - nvidia-gpu-operator
  - gpu-workload
  - kyverno
  - kubevirt-vms

  subscriptions:
    acm:
      name: advanced-cluster-management
      namespace: open-cluster-management
      annotations:
        argocd.argoproj.io/sync-wave: "10"
    eso:
      name: openshift-external-secrets-operator
      namespace: external-secrets-operator
      channel: stable-v1
    sandbox:
      name: sandboxed-containers-operator
      namespace: openshift-sandboxed-containers-operator
      channel: stable
      installPlanApproval: Manual
      csv: sandboxed-containers-operator.v1.13.1
      annotations:
        argocd.argoproj.io/sync-wave: "10"
    trustee:
      name: trustee-operator
      namespace: trustee-operator-system
      channel: stable
      installPlanApproval: Manual
      csv: trustee-operator.v1.2.1
      annotations:
        argocd.argoproj.io/sync-wave: "10"
    cert-manager:
      name: openshift-cert-manager-operator
      namespace: cert-manager-operator
      channel: stable-v1
      annotations:
        argocd.argoproj.io/sync-wave: "10"
    lvm-operator:
      name: lvms-operator
      namespace: openshift-storage
    cnv:
      name: kubevirt-hyperconverged
      namespace: openshift-cnv
      channel: stable
    nfd:
      name: nfd
      namespace: openshift-nfd
      channel: stable
      annotations:
        argocd.argoproj.io/sync-wave: "10"
    # Intel DCAP / device plugins provide TDX quote GENERATION — only needed when
    # the hub itself has Intel TDX hardware. Disabled by the amd-snp hardware profile.
    intel-device-plugins:
      name: intel-device-plugins-operator
      namespace: openshift-operators
      # Required for airgap -- set to your mirrored catalog source name
      # source: cs-certified-operator-index-v4-22
      channel: stable
      annotations:
        argocd.argoproj.io/sync-wave: "10"
    intel-tdx-dcap:
      name: intel-tdx-dcap-operator
      namespace: intel-dcap-operator-system
      # Required for airgap -- set to your mirrored catalog source name
      # source: cs-certified-operator-index-v4-22
      channel: alpha
      annotations:
        argocd.argoproj.io/sync-wave: "10"

  projects:
  - hub
  - vault
  - trustee
  - external-secrets
  - sandbox
  - workloads
  - default

  sharedValueFiles:
  - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml'
  - '/overrides/values-storage-{{ $.Values.global.storageProvider }}.yaml'
  - '/overrides/values-{{ $.Values.global.clusterVersion }}.yaml'
  - '/overrides/values-hw-{{ $.Values.global.hardware.profile }}.yaml'
  - '/overrides/values-baremetal-{{ if $.Values.global.airgap }}airgap{{ else }}connected{{ end }}.yaml'

  extraObjects:
    wait-for-vault-unsealed:
      apiVersion: batch/v1
      kind: Job
      metadata:
        name: wait-for-vault-unsealed
        annotations:
          argocd.argoproj.io/hook: Sync
          argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation
          argocd.argoproj.io/sync-wave: "5"
      spec:
        backoffLimit: 10
        activeDeadlineSeconds: 1800
        template:
          spec:
            restartPolicy: OnFailure
            containers:
              - name: wait-for-vault
                image: registry.redhat.io/openshift4/ose-cli:latest
                command:
                  - /bin/bash
                  - -c
                  - |
                    echo "Waiting for vault to be unsealed..."
                    while true; do
                      if curl -sk https://vault.vault.svc:8200/v1/sys/seal-status | grep -q '"sealed":false'; then
                        echo "Vault unsealed. Waiting 120s for secrets to load..."
                        sleep 120
                        echo "Done. Proceeding."
                        exit 0
                      fi
                      echo "Vault not ready. Retrying in 15s..."
                      sleep 15
                    done

  applications:
    acm:
      name: acm
      namespace: open-cluster-management
      project: hub
      chart: acm
      chartVersion: 0.1.*
      # Required for airgap -- set to your mirrored catalog source name
      # overrides:
      #   - name: acm.mce_operator.source
      #     value: "cs-redhat-operator-index-v4-22"

    vault:
      name: vault
      namespace: vault
      project: vault
      chart: hashicorp-vault
      # Require 0.1.10 or newer within the 0.1.x release series.
      chartVersion: "~0.1.10"
      annotations:
        argocd.argoproj.io/sync-wave: "0"

    secrets-operator:
      name: openshift-external-secrets
      namespace: external-secrets
      project: external-secrets
      chart: openshift-external-secrets
      chartVersion: 0.0.*
      annotations:
        argocd.argoproj.io/sync-wave: "0"

    trustee:
      name: trustee
      namespace: trustee-operator-system
      project: trustee
      chart: trustee
      chartVersion: 0.10.*
      annotations:
        argocd.argoproj.io/sync-wave: "20"
      extraValueFiles:
        - '/overrides/values-trustee.yaml'
      overrides:
        # NOTE: Trustee attestation (quote *verification*) is independent of the hub's
        # own TEE hardware. The hub verifies both TDX and SNP evidence regardless of
        # its own CPU. Node-level quote *generation* infra (Intel DCAP/QGS/SGX,
        # TDX MachineConfig) is controlled by the hardware profile — set
        # global.hardware.profile: intel-tdx for a TDX hub (DCAP deploys) or
        # amd-snp for an AMD hub (DCAP disabled). See values-hw-*.yaml.
        # Enable attestation-gated secret delivery via KBS
        - name: global.coco.secured
          value: "true"
        # TDX attestation — verify TDX evidence from the hub and/or TDX spokes.
        # Verification only; needs collateral (below), not local TDX hardware.
        - name: kbs.tdx.enabled
          value: "true"
        # SNP attestation — verify SNP evidence from AMD SEV-SNP spoke clusters.
        # VCEK certs are loaded via overrides/values-snp-vcek.yaml.
        - name: kbs.snp.enabled
          value: "true"
        # GPU attestation — enables NVIDIA CC GPU reference value verification
        - name: kbs.gpu.enabled
          value: "true"
        # Bare metal mode — uses firmware reference values (not peer-pod PCR measurements)
        - name: kbs.baremetal.enabled
          value: "true"
        # Inject mirror-registry CA into KBS trust store (required for airgap image pulls)
        - name: kbs.registryCa.enabled
          value: "true"
        # TDX DCAP collateral — offline attestation without Intel PCS network access
        - name: kbs.tdx.collateralService
          value: "file:///opt/confidential-containers/attestation-service/tdx/platform_collaterals.json"
        - name: kbs.tdx.collateralMountPath
          value: "/opt/confidential-containers/attestation-service/tdx"
        - name: kbs.tdx.collateralFileName
          value: "platform_collaterals.json"

    storage:
      name: storage
      namespace: openshift-storage
      project: hub
      path: charts/hub/storage
      annotations:
        argocd.argoproj.io/sync-wave: "0"

    baremetal:
      name: baremetal
      namespace: baremetal
      project: hub
      path: charts/all/baremetal
      annotations:
        argocd.argoproj.io/sync-wave: "10"

    sandbox:
      name: sandbox
      namespace: openshift-sandboxed-containers-operator
      project: sandbox
      annotations:
        argocd.argoproj.io/sync-wave: "10"
      chart: sandboxed-containers
      chartVersion: 0.2.*
      overrides:
        - name: enablePeerPods
          value: "false"

    # TDX quote generation (QGS/SGX). Only needed on a TDX hub — disabled by the
    # amd-snp hardware profile. Trustee still verifies TDX evidence from spokes
    # without this (verification uses collateral, not local quote generation).
    intel-dcap:
      name: intel-dcap
      namespace: intel-dcap
      project: hub
      path: charts/all/intel-dcap
      annotations:
        argocd.argoproj.io/sync-wave: "20"
      ignoreDifferences:
        - group: deviceplugin.intel.com
          kind: SgxDevicePlugin
          name: sgxdeviceplugin-sample
          jsonPointers:
            - /spec
        - group: trustedservices.intel.com
          kind: TdxQuoteGenerationService
          name: intel-tdx-dcap
          jsonPointers:
            - /spec
            - /status

    # KubeVirt confidential containers support (experimental)
    # Requires Intel TDX hardware and CNV operator. Gated by chart-level values.
    # Uncomment to enable:
    # kubevirtconfidential:
    #   name: kubevirtconfidential
    #   namespace: openshift-cnv
    #   project: hub
    #   path: charts/all/kubevirtconfidential
    #   annotations:
    #     argocd.argoproj.io/sync-wave: "30"

    # KubeVirt test VMs (experimental)
    # Uncomment to enable:
    # kubevirtvm:
    #   name: kubevirtvm
    #   namespace: kubevirt-vms
    #   project: hub
    #   path: charts/all/kubevirtvm
    #   annotations:
    #     argocd.argoproj.io/sync-wave: "35"
    #   syncPolicy:
    #     automated:
    #       prune: true

    sandbox-policies:
      name: sandbox-policies
      namespace: openshift-sandboxed-containers-operator
      project: sandbox
      chart: sandboxed-policies
      chartVersion: 0.2.*
      annotations:
        argocd.argoproj.io/sync-wave: "20"

    hello-openshift:
      name: hello-openshift
      namespace: hello-openshift
      project: workloads
      path: charts/coco-supported/hello-openshift
      annotations:
        argocd.argoproj.io/sync-wave: "30"
      # Required for airgap — CoCo guest VMs pull images directly, no IDMS/ITMS in the CVM.
      # Uncomment and set MIRROR_REGISTRY_HOST to your mirror registry.
      # overrides:
      #   - name: image
      #     value: "MIRROR_REGISTRY_HOST:8443/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2"
      syncPolicy:
        automated:
          prune: true

    kbs-access-curl:
      name: kbs-access-curl
      namespace: kbs-access
      project: workloads
      path: charts/coco-supported/kbs-access-curl
      annotations:
        argocd.argoproj.io/sync-wave: "30"
      # Required for airgap — CoCo guest VMs pull images directly, no IDMS/ITMS in the CVM.
      # overrides:
      #   - name: image
      #     value: "MIRROR_REGISTRY_HOST:8443/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2"
      #   - name: initImage
      #     value: "MIRROR_REGISTRY_HOST:8443/ubi9/ubi:latest"
      syncPolicy:
        automated:
          prune: true

    kbs-access-sealed:
      name: kbs-access-sealed
      namespace: kbs-access
      project: workloads
      path: charts/coco-supported/kbs-access-sealed
      annotations:
        argocd.argoproj.io/sync-wave: "30"
      # Required for airgap — CoCo guest VMs pull images directly, no IDMS/ITMS in the CVM.
      # overrides:
      #   - name: image
      #     value: "MIRROR_REGISTRY_HOST:8443/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2"
      syncPolicy:
        automated:
          prune: true

    kyverno:
      name: kyverno
      namespace: kyverno
      project: hub
      path: charts/vendor/kyverno
      annotations:
        argocd.argoproj.io/sync-wave: "0"
      syncPolicy:
        automated: {}
        retry:
          limit: 20
        syncOptions:
          - ServerSideApply=true
          - RespectIgnoreDifferences=true
      ignoreDifferences:
        - group: apiextensions.k8s.io
          kind: CustomResourceDefinition
          jqPathExpressions:
            - .metadata.labels
            - .metadata.annotations
      extraValueFiles:
        - '/overrides/values-kyverno.yaml'
      overrides:
        - name: backgroundController.resources.limits.memory
          value: "512Mi"
        - name: backgroundController.resources.requests.memory
          value: "256Mi"

    coco-kyverno-policies:
      name: coco-kyverno-policies
      namespace: openshift-sandboxed-containers-operator
      project: sandbox
      path: charts/all/coco-kyverno-policies
      annotations:
        argocd.argoproj.io/sync-wave: "20"

  imperative:
    serviceAccountCreate: true
    clusterRoleYaml:
      - apiGroups:
        - '*'
        resources:
        - '*'
        verbs:
        - '*'
    jobs:
    - name: init-data-gzipper
      playbook: ansible/init-data-gzipper.yaml
      verbosity: -vvv
      timeout: 3600
    - name: reconcile-kataconfig-gpu
      playbook: ansible/reconcile-kataconfig-gpu.yaml
      verbosity: -vvv
      timeout: 600

  managedClusterGroups:
    baremetal-spoke:
      name: baremetal-spoke
      acmlabels:
      - name: clusterGroup
        value: baremetal-spoke
      helmOverrides:
      - name: clusterGroup.isHubCluster
        value: false
      # Uncomment for HCP spoke clusters (OSC uses DaemonSet mode, MCO templates skipped)
      # - name: global.cluster.hcp
      #   value: true