From 78351e80ac696460d84e1fb9679c6dc6141b4927 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 14 Oct 2025 13:01:14 -0700 Subject: [PATCH 1/6] ci: Use explicit templates for Windows infra --- docs/book/src/developers/development.md | 2 +- scripts/ci-entrypoint.sh | 6 +- ...cluster-template-prow-ci-version-azl3.yaml | 483 +----- .../cluster-template-prow-ci-version-dra.yaml | 442 ------ ...r-template-prow-ci-version-dual-stack.yaml | 194 +-- ...er-template-prow-ci-version-md-and-mp.yaml | 620 +------- ...ster-template-prow-ci-version-windows.yaml | 1341 +++++++++++++++++ .../ci/cluster-template-prow-ci-version.yaml | 483 +----- ...-prow-machine-pool-ci-version-windows.yaml | 1061 +++++++++++++ ...template-prow-machine-pool-ci-version.yaml | 431 ------ ...uster-template-prow-machine-pool-flex.yaml | 368 ----- ...er-template-prow-machine-pool-windows.yaml | 780 ++++++++++ .../cluster-template-prow-machine-pool.yaml | 362 ----- .../ci/cluster-template-prow-topology.yaml | 154 -- .../ci/cluster-template-prow-windows.yaml | 835 ++++++++++ templates/test/ci/cluster-template-prow.yaml | 416 +---- ...ml => machine-pool-win-worker-counts.yaml} | 0 .../kustomization.yaml | 1 - .../prow-ci-version-ipv6/kustomization.yaml | 2 +- .../kustomization.yaml | 10 - .../kustomization.yaml | 74 + ...adm-bootstrap-windows-k8s-ci-binaries.yaml | 35 + .../patches/kubeadm-bootstrap.yaml | 75 + ...e-deployment-ci-version-control-plane.yaml | 26 + ...machine-deployment-ci-version-windows.yaml | 18 + .../machine-deployment-ci-version.yaml | 15 + .../patches/oot-credential-provider-kcp.yaml | 51 + .../patches/oot-credential-provider-win.yaml | 36 + .../patches/oot-credential-provider.yaml | 43 + .../ci/prow-ci-version/kustomization.yaml | 22 - .../kustomization.yaml | 47 + .../kubeadm-bootstrap-k8s-ci-binaries.yaml | 121 ++ ...adm-bootstrap-windows-k8s-ci-binaries.yaml | 70 + .../machine-pool-ci-version-windows.yaml | 18 + .../patches/machine-pool-ci-version.yaml | 14 + .../kustomization.yaml | 14 - .../patches/vmss-flex.yaml | 13 - .../kustomization.yaml | 43 + .../ci/prow-machine-pool/kustomization.yaml | 24 +- templates/test/ci/prow-topology/cluster.yaml | 5 - .../ci/prow-topology/cni-resource-set.yaml | 14 - .../test/ci/prow-topology/kustomization.yaml | 17 - .../cni-resource-set.yaml | 0 .../test/ci/prow-windows/kustomization.yaml | 72 + .../prow-windows/mhc-kubeadmcontrolplane.yaml | 18 + .../{prow => prow-windows}/mhc-windows.yaml | 0 templates/test/ci/prow-windows/mhc.yaml | 15 + templates/test/ci/prow/kustomization.yaml | 46 - .../cluster-template-custom-builds-dra.yaml | 440 ------ ...uster-template-custom-builds-load-dra.yaml | 507 ------- .../cluster-template-custom-builds-load.yaml | 500 +----- ...e-custom-builds-machine-pool-load-dra.yaml | 440 ------ ...plate-custom-builds-machine-pool-load.yaml | 429 ------ ...te-custom-builds-machine-pool-windows.yaml | 1015 +++++++++++++ ...r-template-custom-builds-machine-pool.yaml | 429 ------ ...luster-template-custom-builds-windows.yaml | 1307 ++++++++++++++++ .../dev/cluster-template-custom-builds.yaml | 500 +----- .../kustomization.yaml | 35 + .../patches/custom-builds.yaml | 86 ++ ...-machine-pool-windows-k8s-pr-binaries.yaml | 71 + ...ne-pool-deployment-pr-version-windows.yaml | 14 + .../kustomization.yaml | 19 - .../custom-builds-windows/kustomization.yaml | 69 + ...adm-bootstrap-windows-k8s-pr-binaries.yaml | 58 + .../patches/kubeadm-bootstrap.yaml | 29 + .../kubeadm-controlplane-bootstrap.yaml | 67 + ...machine-deployment-pr-version-windows.yaml | 19 + .../machine-deployment-pr-version.yaml | 50 + .../test/dev/custom-builds/kustomization.yaml | 22 - test/e2e/helpers.go | 2 +- 70 files changed, 7645 insertions(+), 7400 deletions(-) create mode 100644 templates/test/ci/cluster-template-prow-ci-version-windows.yaml create mode 100644 templates/test/ci/cluster-template-prow-machine-pool-ci-version-windows.yaml create mode 100644 templates/test/ci/cluster-template-prow-machine-pool-windows.yaml create mode 100644 templates/test/ci/cluster-template-prow-windows.yaml rename templates/test/ci/patches/{machine-pool-worker-counts.yaml => machine-pool-win-worker-counts.yaml} (100%) create mode 100644 templates/test/ci/prow-ci-version-windows/kustomization.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-control-plane.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-windows.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-kcp.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-win.yaml create mode 100644 templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider.yaml create mode 100644 templates/test/ci/prow-machine-pool-ci-version-windows/kustomization.yaml create mode 100644 templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-k8s-ci-binaries.yaml create mode 100644 templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml create mode 100644 templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version-windows.yaml create mode 100644 templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version.yaml create mode 100644 templates/test/ci/prow-machine-pool-windows/kustomization.yaml delete mode 100644 templates/test/ci/prow-topology/cni-resource-set.yaml rename templates/test/ci/{prow => prow-windows}/cni-resource-set.yaml (100%) create mode 100644 templates/test/ci/prow-windows/kustomization.yaml create mode 100644 templates/test/ci/prow-windows/mhc-kubeadmcontrolplane.yaml rename templates/test/ci/{prow => prow-windows}/mhc-windows.yaml (100%) create mode 100644 templates/test/ci/prow-windows/mhc.yaml create mode 100644 templates/test/dev/cluster-template-custom-builds-machine-pool-windows.yaml create mode 100644 templates/test/dev/cluster-template-custom-builds-windows.yaml create mode 100644 templates/test/dev/custom-builds-machine-pool-windows/kustomization.yaml create mode 100644 templates/test/dev/custom-builds-machine-pool-windows/patches/custom-builds.yaml create mode 100644 templates/test/dev/custom-builds-machine-pool-windows/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml create mode 100644 templates/test/dev/custom-builds-machine-pool-windows/patches/machine-pool-deployment-pr-version-windows.yaml create mode 100644 templates/test/dev/custom-builds-windows/kustomization.yaml create mode 100644 templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml create mode 100644 templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap.yaml create mode 100644 templates/test/dev/custom-builds-windows/patches/kubeadm-controlplane-bootstrap.yaml create mode 100644 templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version-windows.yaml create mode 100644 templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version.yaml diff --git a/docs/book/src/developers/development.md b/docs/book/src/developers/development.md index 35bf7a1edbc..805c6723107 100644 --- a/docs/book/src/developers/development.md +++ b/docs/book/src/developers/development.md @@ -586,7 +586,7 @@ You can optionally set the following variables: | `CI_VERSION` | Provide a custom CI version of Kubernetes (e.g., `v1.25.0-alpha.0.597+aa49dffc7f24dc`). If not specified, this will be determined from the KUBERNETES_VERSION above if it is an unreleased version. If you provide a `CI_VERSION` environment variable, you may not also use `KUBERNETES_VERSION` (above). | | `TEST_CCM` | Build a cluster that uses custom versions of the Azure cloud-provider cloud-controller-manager and node-controller-manager images | | `EXP_MACHINE_POOL` | Use [Machine Pool](../self-managed/machinepools.md) for worker machines. Defaults to true. | -| `TEST_WINDOWS` | Build a cluster that has Windows worker nodes. | +| `TEST_WINDOWS` | Set to "true" to build a cluster that has Windows worker nodes. | | `REGISTRY` | Registry to push any custom k8s images or cloud provider images built. | | `CLUSTER_TEMPLATE` | Use a custom cluster template. It can be a path to a template under templates/, a path on the host or a link. If the value is not set, the script will choose the appropriate cluster template based on existing environment variables. | | `CCM_COUNT` | Set the number of cloud-controller-manager only when `TEST_CCM` is set. Besides it should not be more than control plane Node number. | diff --git a/scripts/ci-entrypoint.sh b/scripts/ci-entrypoint.sh index 2bb0cd19c88..22afb031c36 100755 --- a/scripts/ci-entrypoint.sh +++ b/scripts/ci-entrypoint.sh @@ -117,7 +117,7 @@ setup() { export EXP_CLUSTER_RESOURCE_SET="true" # TODO figure out a better way to account for expected Windows node count - if [[ -n "${TEST_WINDOWS:-}" ]]; then + if [[ "${TEST_WINDOWS:-}" == "true" ]]; then export WINDOWS_WORKER_MACHINE_COUNT="${WINDOWS_WORKER_MACHINE_COUNT:-2}" fi } @@ -139,6 +139,10 @@ select_cluster_template() { export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/custom-builds/custom-builds-machine-pool}" fi fi + + if [[ "${TEST_WINDOWS:-}" == "true" ]]; then + export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/.yaml/-windows.yaml}" + fi } create_cluster() { diff --git a/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml index 2d988188f15..fd7b64be451 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-azl3.yaml @@ -4,9 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled metrics-server: enabled name: ${CLUSTER_NAME} namespace: default @@ -59,8 +56,7 @@ spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - feature-gates: ${K8S_FEATURE_GATES:-""} + extraArgs: {} timeoutForControlPlane: 20m controllerManager: extraArgs: @@ -526,193 +522,6 @@ spec: verbosity: 5 --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/oot-cred-provider.ps1 - - powershell C:/replace-ci-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -747,20 +556,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity metadata: @@ -774,34 +569,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -959,254 +726,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: diff --git a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml index 8d2e72733b0..3a65538cddb 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dra.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dra.yaml @@ -4,10 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -473,154 +469,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - identity: UserAssigned - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - path: C:/replace-k8s-binaries.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - echo "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - #!/bin/bash - - echo "enabling containerd CDI plugin" - sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a\ enable_cdi = true' /etc/containerd/config.toml - systemctl restart containerd - owner: root:root - path: /tmp/containerd-config.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-"DynamicResourceAllocation=true"} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - bash -c /tmp/containerd-config.sh - - powershell c:/create-external-network.ps1 - - powershell C:/replace-k8s-binaries.ps1 - - powershell C:/oot-cred-provider.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -655,48 +503,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -851,251 +657,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 86c8531caba..03027d02df4 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -4,10 +4,7 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico-dual-stack - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: disabled - csi-proxy: disabled - metrics-server: disabled + metrics-server: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -73,8 +70,7 @@ spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - feature-gates: ${K8S_FEATURE_GATES:-""} + extraArgs: {} timeoutForControlPlane: 20m controllerManager: extraArgs: @@ -501,34 +497,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -598,164 +566,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: diff --git a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml index cfcbef73f84..238560d9c9f 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-md-and-mp.yaml @@ -4,9 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled metrics-server: enabled name: ${CLUSTER_NAME} namespace: default @@ -59,8 +56,7 @@ spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - feature-gates: ${K8S_FEATURE_GATES:-""} + extraArgs: {} timeoutForControlPlane: 20m controllerManager: extraArgs: @@ -431,193 +427,6 @@ spec: verbosity: 5 --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/oot-cred-provider.ps1 - - powershell C:/replace-ci-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -652,20 +461,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity metadata: @@ -679,34 +474,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -862,254 +629,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: @@ -1510,140 +1029,3 @@ spec: preKubeadmCommands: - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - identity: UserAssigned - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - path: C:/replace-k8s-binaries.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - echo "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - - powershell C:/replace-k8s-binaries.ps1 - - powershell C:/oot-cred-provider.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml new file mode 100644 index 00000000000..9c1fdc31ea8 --- /dev/null +++ b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml @@ -0,0 +1,1341 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + metrics-server: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + feature-gates: ${K8S_FEATURE_GATES:-""} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + selector: {} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + spec: + identity: UserAssigned + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" + if($$CONTAINERD_URL -ne ""){ + # Kubelet service depends on contianerd service so make a best effort attempt to stop it + Stop-Service kubelet -Force -ErrorAction SilentlyContinue + Stop-Service containerd -Force + echo "downloading containerd: $$CONTAINERD_URL" + curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" + tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 + + Start-Service containerd + } + + containerd.exe --version + containerd-shim-runhcs-v1.exe --version + path: C:/replace-containerd.ps1 + permissions: "0744" + - content: | + mkdir -Force c:/localdumps + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f + # Enable sftp so we can copy crash dump files during log collection of stfp + $sshd_config = "$env:ProgramData\ssh\sshd_config" + if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } + Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" + sc.exe stop sshd + sc.exe start sshd + path: C:/collect-hns-crashes.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + Write-Host "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$CI_VERSION="${CI_VERSION}" + if($$CI_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-ci-binaries.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell C:/create-temp-folder.ps1 + - powershell C:/replace-containerd.ps1 + - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 + - powershell C:/replace-ci-binaries.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: v1 +data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + priorityClassName: system-node-critical + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: metrics-server-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + metrics-server: enabled + resources: + - kind: ConfigMap + name: metrics-server-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + metrics-server: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader + rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - emptyDir: {} + name: tmp-dir + --- + apiVersion: apiregistration.k8s.io/v1 + kind: APIService + metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io + spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: metrics-server-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 9c1fdc31ea8..cd1243b24c9 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -4,9 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled metrics-server: enabled name: ${CLUSTER_NAME} namespace: default @@ -59,8 +56,7 @@ spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - feature-gates: ${K8S_FEATURE_GATES:-""} + extraArgs: {} timeoutForControlPlane: 20m controllerManager: extraArgs: @@ -431,193 +427,6 @@ spec: verbosity: 5 --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/oot-cred-provider.ps1 - - powershell C:/replace-ci-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -652,20 +461,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity metadata: @@ -679,34 +474,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -862,254 +629,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version-windows.yaml new file mode 100644 index 00000000000..5d82db90e72 --- /dev/null +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version-windows.yaml @@ -0,0 +1,1061 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: {} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + identity: UserAssigned + location: ${AZURE_LOCATION} + strategy: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/kubeadm-bootstrap.sh +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + identity: UserAssigned + location: ${AZURE_LOCATION} + template: + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$CI_VERSION="${CI_VERSION}" + if($$CI_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + path: C:/replace-k8s-binaries.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + echo "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell c:/create-external-network.ps1 + - powershell C:/replace-k8s-binaries.ps1 + - powershell C:/oot-cred-provider.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: v1 +data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + priorityClassName: system-node-critical + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 5d82db90e72..9ba2d58298d 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -4,10 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -444,143 +440,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - identity: UserAssigned - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - path: C:/replace-k8s-binaries.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - echo "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - - powershell C:/replace-k8s-binaries.ps1 - - powershell C:/oot-cred-provider.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -615,48 +474,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -811,251 +628,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 3caa4ba8fe7..865ad9ec647 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -4,10 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -232,88 +228,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - orchestrationMode: Flexible - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 0 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -348,48 +262,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -544,243 +416,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml new file mode 100644 index 00000000000..9347e82c74e --- /dev/null +++ b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml @@ -0,0 +1,780 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: {} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + verbosity: 10 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + strategy: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + location: ${AZURE_LOCATION} + template: + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell c:/create-external-network.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: v1 +data: + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + priorityClassName: system-node-critical + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 9347e82c74e..fa8e8891dbe 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -4,10 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -232,82 +228,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -342,48 +262,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -538,243 +416,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-topology.yaml b/templates/test/ci/cluster-template-prow-topology.yaml index c1b2f3d37cf..d03ce9c3116 100644 --- a/templates/test/ci/cluster-template-prow-topology.yaml +++ b/templates/test/ci/cluster-template-prow-topology.yaml @@ -4,7 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled name: ${CLUSTER_NAME} @@ -25,8 +24,6 @@ spec: value: ${AZURE_CONTROL_PLANE_MACHINE_TYPE:-""} - name: workerMachineType value: ${AZURE_NODE_MACHINE_TYPE:-""} - - name: workerMachineTypeWin - value: ${AZURE_NODE_MACHINE_TYPE:-""} - name: sshPublicKey value: ${AZURE_SSH_PUBLIC_KEY_B64:-""} - name: buildProvenance @@ -49,37 +46,6 @@ spec: - class: ${CLUSTER_NAME}-worker name: md-0 replicas: ${WORKER_MACHINE_COUNT} - - class: ${CLUSTER_NAME}-worker-win - name: md-win - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce --- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy @@ -235,123 +201,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml new file mode 100644 index 00000000000..269971fe6a9 --- /dev/null +++ b/templates/test/ci/cluster-template-prow-windows.yaml @@ -0,0 +1,835 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + feature-gates: ${K8S_FEATURE_GATES:-""} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + verbosity: 10 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + identity: UserAssigned + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + selector: {} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + spec: + identity: UserAssigned + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" + if($$CONTAINERD_URL -ne ""){ + # Kubelet service depends on contianerd service so make a best effort attempt to stop it + Stop-Service kubelet -Force -ErrorAction SilentlyContinue + Stop-Service containerd -Force + echo "downloading containerd: $$CONTAINERD_URL" + curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" + tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 + + Start-Service containerd + } + + containerd.exe --version + containerd-shim-runhcs-v1.exe --version + path: C:/replace-containerd.ps1 + permissions: "0744" + - content: | + mkdir -Force c:/localdumps + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f + # Enable sftp so we can copy crash dump files during log collection of stfp + $sshd_config = "$env:ProgramData\ssh\sshd_config" + if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } + Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" + sc.exe stop sshd + sc.exe start sshd + path: C:/collect-hns-crashes.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + feature-gates: ${NODE_FEATURE_GATES:-""} + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell C:/create-temp-folder.ps1 + - powershell C:/replace-containerd.ps1 + - powershell C:/collect-hns-crashes.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: v1 +data: + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + priorityClassName: system-node-critical + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 269971fe6a9..08f542bb060 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -4,9 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -58,8 +55,7 @@ spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - feature-gates: ${K8S_FEATURE_GATES:-""} + extraArgs: {} timeoutForControlPlane: 20m controllerManager: extraArgs: @@ -222,134 +218,6 @@ spec: preKubeadmCommands: [] --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -384,20 +252,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity metadata: @@ -411,34 +265,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -593,243 +419,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/patches/machine-pool-worker-counts.yaml b/templates/test/ci/patches/machine-pool-win-worker-counts.yaml similarity index 100% rename from templates/test/ci/patches/machine-pool-worker-counts.yaml rename to templates/test/ci/patches/machine-pool-win-worker-counts.yaml diff --git a/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml b/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml index 4d01fd9ab39..cc756cbbd21 100644 --- a/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml +++ b/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml @@ -27,7 +27,6 @@ patches: - path: ../../../flavors/dual-stack/patches/kubeadm-controlplane.yaml - path: ../prow-dual-stack/patches/cluster-label-calico-dual-stack.yaml - path: patches/machine-deployment.yaml -- path: ../patches/windows-addons-disabled.yaml sortOptions: order: fifo diff --git a/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml b/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml index 7298ad5aaab..abf6601651e 100644 --- a/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml +++ b/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: -- ../prow-ci-version +- ../prow-ci-version-windows - ../../../addons/cluster-api-helm/calico-ipv6.yaml patches: - patch: | diff --git a/templates/test/ci/prow-ci-version-md-and-mp/kustomization.yaml b/templates/test/ci/prow-ci-version-md-and-mp/kustomization.yaml index 2f608fe5bac..6e5dfc7604a 100644 --- a/templates/test/ci/prow-ci-version-md-and-mp/kustomization.yaml +++ b/templates/test/ci/prow-ci-version-md-and-mp/kustomization.yaml @@ -4,15 +4,7 @@ namespace: default resources: - ../prow-ci-version - ../../../flavors/machinepool/machine-pool-deployment.yaml -- ../../../flavors/machinepool-windows/machine-pool-deployment-windows.yaml patches: -- path: ../prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfig - name: .*-mp-win - namespace: default - version: v1beta1 - path: ../prow-machine-pool-ci-version/patches/kubeadm-bootstrap-k8s-ci-binaries.yaml target: group: bootstrap.cluster.x-k8s.io @@ -21,8 +13,6 @@ patches: namespace: default version: v1beta1 - path: ../prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml -- path: ../patches/machine-pool-worker-counts.yaml -- path: ../prow-machine-pool-ci-version/patches/machine-pool-ci-version-windows.yaml - path: ../patches/uami-mp-0.yaml sortOptions: order: fifo diff --git a/templates/test/ci/prow-ci-version-windows/kustomization.yaml b/templates/test/ci/prow-ci-version-windows/kustomization.yaml new file mode 100644 index 00000000000..58efc34ab48 --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/kustomization.yaml @@ -0,0 +1,74 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../prow-windows +- ../../../addons/metrics-server/metrics-server-resource-set.yaml +patches: +- path: patches/oot-credential-provider.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 +- path: patches/oot-credential-provider-win.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + version: v1beta1 +- path: patches/oot-credential-provider-kcp.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: patches/kubeadm-bootstrap.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 +- path: patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + version: v1beta1 +- path: ../patches/control-plane-kubeadm-boostrap-ci-version.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: ../patches/control-plane-image-ci-version.yaml +- path: ../patches/controller-manager.yaml +- path: ../patches/windows-feature-gates.yaml +- path: ../patches/windows-containerd-labels.yaml +- path: ../patches/windows-machine-deployment-replicas.yaml +- path: patches/machine-deployment-ci-version.yaml +- path: patches/machine-deployment-ci-version-windows.yaml +- path: patches/machine-deployment-ci-version-control-plane.yaml +- path: ../patches/metrics-server-enabled-cluster.yaml +- path: ../patches/uami-md-0.yaml +configMapGenerator: +- behavior: merge + files: + - kube-proxy-patch=../patches/windows-kubeproxy-ci.yaml + name: cni-${CLUSTER_NAME}-calico-windows +- files: + - metrics-server=../../../addons/metrics-server/metrics-server.yaml + name: metrics-server-${CLUSTER_NAME} +generatorOptions: + annotations: + note: generated + disableNameSuffixHash: true + labels: + type: generated + +sortOptions: + order: fifo diff --git a/templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml b/templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml new file mode 100644 index 00000000000..5d2b4f53d57 --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml @@ -0,0 +1,35 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$CI_VERSION="${CI_VERSION}" + if($$CI_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-ci-binaries.ps1 + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/replace-ci-binaries.ps1 diff --git a/templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap.yaml b/templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap.yaml new file mode 100644 index 00000000000..743710d5ec6 --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/kubeadm-bootstrap.yaml @@ -0,0 +1,75 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + path: /tmp/kubeadm-bootstrap.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + bash -c /tmp/kubeadm-bootstrap.sh +- op: add + path: /spec/template/spec/verbosity + value: + 5 diff --git a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-control-plane.yaml b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-control-plane.yaml new file mode 100644 index 00000000000..8d684ecd158 --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-control-plane.yaml @@ -0,0 +1,26 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} diff --git a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-windows.yaml b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-windows.yaml new file mode 100644 index 00000000000..a0e8936019e --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version-windows.yaml @@ -0,0 +1,18 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-win +spec: + template: + spec: + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # the binaries and images will get replaced to the desired version by the pre-kubeadm script. + marketplace: + publisher: cncf-upstream + offer: capi-windows + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: "latest" + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} diff --git a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml new file mode 100644 index 00000000000..65eb8a4bb3f --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml @@ -0,0 +1,15 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script in this template. + marketplace: + publisher: cncf-upstream + offer: capi + sku: ubuntu-2204-gen1 + version: "latest" diff --git a/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-kcp.yaml b/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-kcp.yaml new file mode 100644 index 00000000000..1351014d846 --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-kcp.yaml @@ -0,0 +1,51 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands/- + value: + bash -c /tmp/oot-cred-provider.sh +- op: add + path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml +- op: add + path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-win.yaml b/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-win.yaml new file mode 100644 index 00000000000..56264a81a43 --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider-win.yaml @@ -0,0 +1,36 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + Write-Host "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider.yaml b/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider.yaml new file mode 100644 index 00000000000..3da99ef17d0 --- /dev/null +++ b/templates/test/ci/prow-ci-version-windows/patches/oot-credential-provider.yaml @@ -0,0 +1,43 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + bash -c /tmp/oot-cred-provider.sh +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-ci-version/kustomization.yaml b/templates/test/ci/prow-ci-version/kustomization.yaml index 195dbc5d9ec..c5b33dfa851 100644 --- a/templates/test/ci/prow-ci-version/kustomization.yaml +++ b/templates/test/ci/prow-ci-version/kustomization.yaml @@ -12,13 +12,6 @@ patches: name: .*-md-0 namespace: default version: v1beta1 -- path: patches/oot-credential-provider-win.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - version: v1beta1 - path: patches/oot-credential-provider-kcp.yaml target: group: controlplane.cluster.x-k8s.io @@ -32,13 +25,6 @@ patches: name: .*-md-0 namespace: default version: v1beta1 -- path: patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - version: v1beta1 - path: ../patches/control-plane-kubeadm-boostrap-ci-version.yaml target: group: controlplane.cluster.x-k8s.io @@ -47,19 +33,11 @@ patches: version: v1beta1 - path: ../patches/control-plane-image-ci-version.yaml - path: ../patches/controller-manager.yaml -- path: ../patches/windows-feature-gates.yaml -- path: ../patches/windows-containerd-labels.yaml -- path: ../patches/windows-machine-deployment-replicas.yaml - path: patches/machine-deployment-ci-version.yaml -- path: patches/machine-deployment-ci-version-windows.yaml - path: patches/machine-deployment-ci-version-control-plane.yaml - path: ../patches/metrics-server-enabled-cluster.yaml - path: ../patches/uami-md-0.yaml configMapGenerator: -- behavior: merge - files: - - kube-proxy-patch=../patches/windows-kubeproxy-ci.yaml - name: cni-${CLUSTER_NAME}-calico-windows - files: - metrics-server=../../../addons/metrics-server/metrics-server.yaml name: metrics-server-${CLUSTER_NAME} diff --git a/templates/test/ci/prow-machine-pool-ci-version-windows/kustomization.yaml b/templates/test/ci/prow-machine-pool-ci-version-windows/kustomization.yaml new file mode 100644 index 00000000000..488e5547217 --- /dev/null +++ b/templates/test/ci/prow-machine-pool-ci-version-windows/kustomization.yaml @@ -0,0 +1,47 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../prow-machine-pool-windows +patches: +- path: ../prow-ci-version/patches/oot-credential-provider-kcp.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: ../patches/control-plane-kubeadm-boostrap-ci-version.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + namespace: default + version: v1beta1 +- path: patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfig + name: .*-mp-win + namespace: default + version: v1beta1 +- path: patches/kubeadm-bootstrap-k8s-ci-binaries.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfig + name: .*-mp-0 + namespace: default + version: v1beta1 +- path: ../patches/control-plane-image-ci-version.yaml +- path: ../patches/controller-manager.yaml +- path: patches/machine-pool-ci-version.yaml +- path: ../patches/machine-pool-win-worker-counts.yaml +- path: patches/machine-pool-ci-version-windows.yaml +- path: ../patches/uami-mp-0.yaml +configMapGenerator: +- behavior: merge + files: + - kube-proxy-patch=../patches/windows-kubeproxy-ci.yaml + name: cni-${CLUSTER_NAME}-calico-windows + +sortOptions: + order: fifo diff --git a/templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-k8s-ci-binaries.yaml b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-k8s-ci-binaries.yaml new file mode 100644 index 00000000000..a52202eab37 --- /dev/null +++ b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-k8s-ci-binaries.yaml @@ -0,0 +1,121 @@ +- op: add + path: /spec/files/0 + value: + path: /tmp/kubeadm-bootstrap.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + systemctl restart kubelet + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" +- op: add + path: /spec/files/0 + value: + path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi +- op: test + path: /spec/preKubeadmCommands + value: null +- op: add + path: /spec/preKubeadmCommands + value: [] +- op: add + path: /spec/preKubeadmCommands/- + value: + bash -c /tmp/oot-cred-provider.sh +- op: add + path: /spec/preKubeadmCommands/- + value: + bash -c /tmp/kubeadm-bootstrap.sh +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml new file mode 100644 index 00000000000..64a8700c90f --- /dev/null +++ b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml @@ -0,0 +1,70 @@ +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$CI_VERSION="${CI_VERSION}" + if($$CI_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + path: C:/replace-k8s-binaries.ps1 + permissions: "0744" +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + echo "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/replace-k8s-binaries.ps1 +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version-windows.yaml b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version-windows.yaml new file mode 100644 index 00000000000..07beec81a4c --- /dev/null +++ b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version-windows.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win +spec: + template: + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script in this template. + marketplace: + publisher: cncf-upstream + offer: capi-windows + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} diff --git a/templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version.yaml b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version.yaml new file mode 100644 index 00000000000..74718bda519 --- /dev/null +++ b/templates/test/ci/prow-machine-pool-ci-version-windows/patches/machine-pool-ci-version.yaml @@ -0,0 +1,14 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 +spec: + template: + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi + sku: ubuntu-2204-gen1 + version: latest diff --git a/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml b/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml index 37a63e79205..d3228681de6 100644 --- a/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml @@ -17,13 +17,6 @@ patches: name: .*-control-plane namespace: default version: v1beta1 -- path: patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfig - name: .*-mp-win - namespace: default - version: v1beta1 - path: patches/kubeadm-bootstrap-k8s-ci-binaries.yaml target: group: bootstrap.cluster.x-k8s.io @@ -34,14 +27,7 @@ patches: - path: ../patches/control-plane-image-ci-version.yaml - path: ../patches/controller-manager.yaml - path: patches/machine-pool-ci-version.yaml -- path: ../patches/machine-pool-worker-counts.yaml -- path: patches/machine-pool-ci-version-windows.yaml - path: ../patches/uami-mp-0.yaml -configMapGenerator: -- behavior: merge - files: - - kube-proxy-patch=../patches/windows-kubeproxy-ci.yaml - name: cni-${CLUSTER_NAME}-calico-windows sortOptions: order: fifo diff --git a/templates/test/ci/prow-machine-pool-flex/patches/vmss-flex.yaml b/templates/test/ci/prow-machine-pool-flex/patches/vmss-flex.yaml index b772d9dd2b5..50644f96e89 100644 --- a/templates/test/ci/prow-machine-pool-flex/patches/vmss-flex.yaml +++ b/templates/test/ci/prow-machine-pool-flex/patches/vmss-flex.yaml @@ -10,16 +10,3 @@ spec: maxSurge: 0% maxUnavailable: 0 deletePolicy: ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-win" -spec: - orchestrationMode: Flexible - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0% - maxUnavailable: 0 - deletePolicy: diff --git a/templates/test/ci/prow-machine-pool-windows/kustomization.yaml b/templates/test/ci/prow-machine-pool-windows/kustomization.yaml new file mode 100644 index 00000000000..bfd51eb23f8 --- /dev/null +++ b/templates/test/ci/prow-machine-pool-windows/kustomization.yaml @@ -0,0 +1,43 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../../../flavors/machinepool-windows +- ../prow-windows/mhc-kubeadmcontrolplane.yaml +- ../prow-windows/mhc.yaml +- ../prow-windows/cni-resource-set.yaml +- ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml +- ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml +- ../../../addons/cluster-api-helm/calico.yaml +- ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml +configMapGenerator: +- files: + - windows-cni=../../../addons/windows/calico/calico.yaml + - proxy=../../../addons/windows/calico/kube-proxy-windows.yaml + name: cni-${CLUSTER_NAME}-calico-windows +- files: + - csi-proxy=../../../addons/windows/csi-proxy/csi-proxy.yaml + name: csi-proxy-addon +- files: + - containerd-windows-logger=../../../addons/windows/containerd-logging/containerd-logger.yaml + name: containerd-logger-${CLUSTER_NAME} +generatorOptions: + annotations: + note: generated + disableNameSuffixHash: true + labels: + type: generated +patches: +- path: ../patches/azuremachinepool-vmextension.yaml +- path: ../patches/tags.yaml +- path: ../patches/controller-manager.yaml +- path: ../patches/machine-pool-win-worker-counts.yaml +- path: ../patches/windows-containerd-labels.yaml +- path: ../patches/cluster-label-calico.yaml +- path: ../patches/cluster-label-cloud-provider-azure.yaml +- path: ../patches/mhc-machinepool.yaml + +sortOptions: + order: fifo diff --git a/templates/test/ci/prow-machine-pool/kustomization.yaml b/templates/test/ci/prow-machine-pool/kustomization.yaml index c7458761926..2117d525a96 100644 --- a/templates/test/ci/prow-machine-pool/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool/kustomization.yaml @@ -2,39 +2,17 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: -- ../../../flavors/machinepool-windows +- ../../../flavors/machinepool - ../prow/mhc-kubeadmcontrolplane.yaml - ../prow/mhc.yaml -- ../prow/cni-resource-set.yaml -- ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml -- ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml -configMapGenerator: -- files: - - windows-cni=../../../addons/windows/calico/calico.yaml - - proxy=../../../addons/windows/calico/kube-proxy-windows.yaml - name: cni-${CLUSTER_NAME}-calico-windows -- files: - - csi-proxy=../../../addons/windows/csi-proxy/csi-proxy.yaml - name: csi-proxy-addon -- files: - - containerd-windows-logger=../../../addons/windows/containerd-logging/containerd-logger.yaml - name: containerd-logger-${CLUSTER_NAME} -generatorOptions: - annotations: - note: generated - disableNameSuffixHash: true - labels: - type: generated patches: - path: ../patches/azuremachinepool-vmextension.yaml - path: ../patches/tags.yaml - path: ../patches/controller-manager.yaml -- path: ../patches/machine-pool-worker-counts.yaml -- path: ../patches/windows-containerd-labels.yaml - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml - path: ../patches/mhc-machinepool.yaml diff --git a/templates/test/ci/prow-topology/cluster.yaml b/templates/test/ci/prow-topology/cluster.yaml index 585106de8df..426ab94733d 100644 --- a/templates/test/ci/prow-topology/cluster.yaml +++ b/templates/test/ci/prow-topology/cluster.yaml @@ -10,9 +10,6 @@ spec: - class: ${CLUSTER_NAME}-worker name: md-0 replicas: ${WORKER_MACHINE_COUNT} - - class: ${CLUSTER_NAME}-worker-win - name: md-win - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} variables: - name: subscriptionID value: ${AZURE_SUBSCRIPTION_ID} @@ -20,8 +17,6 @@ spec: value: ${AZURE_CONTROL_PLANE_MACHINE_TYPE:-""} - name: workerMachineType value: ${AZURE_NODE_MACHINE_TYPE:-""} - - name: workerMachineTypeWin - value: ${AZURE_NODE_MACHINE_TYPE:-""} - name: sshPublicKey value: ${AZURE_SSH_PUBLIC_KEY_B64:-""} - name: buildProvenance diff --git a/templates/test/ci/prow-topology/cni-resource-set.yaml b/templates/test/ci/prow-topology/cni-resource-set.yaml deleted file mode 100644 index 31af8f58ca2..00000000000 --- a/templates/test/ci/prow-topology/cni-resource-set.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce diff --git a/templates/test/ci/prow-topology/kustomization.yaml b/templates/test/ci/prow-topology/kustomization.yaml index 32b08e6fc30..d7458127b2c 100644 --- a/templates/test/ci/prow-topology/kustomization.yaml +++ b/templates/test/ci/prow-topology/kustomization.yaml @@ -3,28 +3,11 @@ kind: Kustomization namespace: default resources: - ../../../flavors/topology/cluster.yaml -- cni-resource-set.yaml -- ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml -configMapGenerator: -- files: - - windows-cni=../../../addons/windows/calico/calico.yaml - - proxy=../../../addons/windows/calico/kube-proxy-windows.yaml - name: cni-${CLUSTER_NAME}-calico-windows -- files: - - csi-proxy=../../../addons/windows/csi-proxy/csi-proxy.yaml - name: csi-proxy-addon -generatorOptions: - annotations: - note: generated - disableNameSuffixHash: true - labels: - type: generated patches: -- path: ../patches/windows-containerd-labels.yaml - path: cluster.yaml - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml diff --git a/templates/test/ci/prow/cni-resource-set.yaml b/templates/test/ci/prow-windows/cni-resource-set.yaml similarity index 100% rename from templates/test/ci/prow/cni-resource-set.yaml rename to templates/test/ci/prow-windows/cni-resource-set.yaml diff --git a/templates/test/ci/prow-windows/kustomization.yaml b/templates/test/ci/prow-windows/kustomization.yaml new file mode 100644 index 00000000000..81f7b28fb41 --- /dev/null +++ b/templates/test/ci/prow-windows/kustomization.yaml @@ -0,0 +1,72 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../../../flavors/base +- ../../../flavors/default/machine-deployment.yaml +- ../../../flavors/windows/machine-deployment-windows.yaml +- mhc-kubeadmcontrolplane.yaml +- mhc.yaml +- cni-resource-set.yaml +- ../../../azure-cluster-identity +- ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml +- ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml +- ../../../addons/cluster-api-helm/calico.yaml +- ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml +patches: +- path: ../patches/windows-tmp-folder.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + version: v1beta1 +- path: ../patches/kubeadm-bootstrap-windows-containerd.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + version: v1beta1 +- path: ../patches/windows-collect-hns-crashes.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + version: v1beta1 +- path: ../patches/tags.yaml +- path: ../patches/mhc.yaml +- path: ../patches/controller-manager.yaml +- path: ../patches/windows-machine-deployment-replicas.yaml +- path: ../../../azure-cluster-identity/azurecluster-identity-ref.yaml +- path: ../patches/azuremachinetemplate-vmextension.yaml +- path: ../patches/azuremachinetemplate-monitoring.yaml +- path: ../patches/windows-feature-gates.yaml +- path: ../patches/windows-containerd-labels.yaml +- path: ../patches/windows-server-version.yaml +- path: ../patches/cluster-label-calico.yaml +- path: ../patches/cluster-label-cloud-provider-azure.yaml +- path: ../patches/uami-md-0.yaml +configMapGenerator: +- files: + - windows-cni=../../../addons/windows/calico/calico.yaml + - proxy=../../../addons/windows/calico/kube-proxy-windows.yaml + name: cni-${CLUSTER_NAME}-calico-windows +- files: + - csi-proxy=../../../addons/windows/csi-proxy/csi-proxy.yaml + name: csi-proxy-addon +- files: + - containerd-windows-logger=../../../addons/windows/containerd-logging/containerd-logger.yaml + name: containerd-logger-${CLUSTER_NAME} +generatorOptions: + annotations: + note: generated + disableNameSuffixHash: true + labels: + type: generated + +sortOptions: + order: fifo diff --git a/templates/test/ci/prow-windows/mhc-kubeadmcontrolplane.yaml b/templates/test/ci/prow-windows/mhc-kubeadmcontrolplane.yaml new file mode 100644 index 00000000000..97b7d7a2c06 --- /dev/null +++ b/templates/test/ci/prow-windows/mhc-kubeadmcontrolplane.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + clusterName: "${CLUSTER_NAME}" + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: 300s + - type: Ready + status: "False" + timeout: 300s diff --git a/templates/test/ci/prow/mhc-windows.yaml b/templates/test/ci/prow-windows/mhc-windows.yaml similarity index 100% rename from templates/test/ci/prow/mhc-windows.yaml rename to templates/test/ci/prow-windows/mhc-windows.yaml diff --git a/templates/test/ci/prow-windows/mhc.yaml b/templates/test/ci/prow-windows/mhc.yaml new file mode 100644 index 00000000000..dec016d9aff --- /dev/null +++ b/templates/test/ci/prow-windows/mhc.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "${CLUSTER_NAME}-mhc-0" +spec: + clusterName: "${CLUSTER_NAME}" + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: "pool1" + unhealthyConditions: + - type: E2ENodeUnhealthy + status: "True" + timeout: 30s \ No newline at end of file diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index 81f7b28fb41..7ccb8b62c2d 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -4,69 +4,23 @@ namespace: default resources: - ../../../flavors/base - ../../../flavors/default/machine-deployment.yaml -- ../../../flavors/windows/machine-deployment-windows.yaml - mhc-kubeadmcontrolplane.yaml - mhc.yaml -- cni-resource-set.yaml - ../../../azure-cluster-identity -- ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml -- ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patches: -- path: ../patches/windows-tmp-folder.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - version: v1beta1 -- path: ../patches/kubeadm-bootstrap-windows-containerd.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - version: v1beta1 -- path: ../patches/windows-collect-hns-crashes.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - version: v1beta1 - path: ../patches/tags.yaml - path: ../patches/mhc.yaml - path: ../patches/controller-manager.yaml -- path: ../patches/windows-machine-deployment-replicas.yaml - path: ../../../azure-cluster-identity/azurecluster-identity-ref.yaml - path: ../patches/azuremachinetemplate-vmextension.yaml - path: ../patches/azuremachinetemplate-monitoring.yaml -- path: ../patches/windows-feature-gates.yaml -- path: ../patches/windows-containerd-labels.yaml -- path: ../patches/windows-server-version.yaml - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml - path: ../patches/uami-md-0.yaml -configMapGenerator: -- files: - - windows-cni=../../../addons/windows/calico/calico.yaml - - proxy=../../../addons/windows/calico/kube-proxy-windows.yaml - name: cni-${CLUSTER_NAME}-calico-windows -- files: - - csi-proxy=../../../addons/windows/csi-proxy/csi-proxy.yaml - name: csi-proxy-addon -- files: - - containerd-windows-logger=../../../addons/windows/containerd-logging/containerd-logger.yaml - name: containerd-logger-${CLUSTER_NAME} -generatorOptions: - annotations: - note: generated - disableNameSuffixHash: true - labels: - type: generated sortOptions: order: fifo diff --git a/templates/test/dev/cluster-template-custom-builds-dra.yaml b/templates/test/dev/cluster-template-custom-builds-dra.yaml index 51bdfb29e27..ca823f9724d 100644 --- a/templates/test/dev/cluster-template-custom-builds-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-dra.yaml @@ -4,10 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -429,152 +425,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - #!/bin/bash - - echo "enabling containerd CDI plugin" - sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a\ enable_cdi = true' /etc/containerd/config.toml - systemctl restart containerd - owner: root:root - path: /tmp/containerd-config.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-"DynamicResourceAllocation=true"} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - bash -c /tmp/containerd-config.sh - - powershell c:/create-external-network.ps1 - - powershell C:/replace-pr-binaries.ps1 - - powershell C:/oot-cred-provider.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -609,48 +459,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -805,251 +613,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml index c454cdfe288..e67db55a7f4 100644 --- a/templates/test/dev/cluster-template-custom-builds-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load-dra.yaml @@ -5,9 +5,6 @@ metadata: azuredisk-csi: "true" cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled metrics-server: enabled storageclass: "true" name: ${CLUSTER_NAME} @@ -414,220 +411,6 @@ spec: - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - Write-Host "Installing Azure CLI" - $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - Write-Host "Version Azure CLI:" - az version - path: C:/install-az-cli.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - # Install Binaries - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - echo "running az login via identity" - az login --identity - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - - content: | - #!/bin/bash - - echo "enabling containerd CDI plugin" - sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a\ enable_cdi = true' /etc/containerd/config.toml - systemctl restart containerd - owner: root:root - path: /tmp/containerd-config.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-"DynamicResourceAllocation=true"} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - bash -c /tmp/containerd-config.sh - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/oot-cred-provider.ps1 - - powershell C:/install-az-cli.ps1 - - powershell C:/replace-pr-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -662,20 +445,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity metadata: @@ -689,34 +458,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -872,254 +613,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: diff --git a/templates/test/dev/cluster-template-custom-builds-load.yaml b/templates/test/dev/cluster-template-custom-builds-load.yaml index 4a5039fa0a5..e1b66843d6a 100644 --- a/templates/test/dev/cluster-template-custom-builds-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-load.yaml @@ -5,9 +5,6 @@ metadata: azuredisk-csi: "true" cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled metrics-server: enabled storageclass: "true" name: ${CLUSTER_NAME} @@ -63,8 +60,7 @@ spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - feature-gates: ${K8S_FEATURE_GATES:-""} + extraArgs: {} timeoutForControlPlane: 20m controllerManager: extraArgs: @@ -386,210 +382,6 @@ spec: - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - Write-Host "Installing Azure CLI" - $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - Write-Host "Version Azure CLI:" - az version - path: C:/install-az-cli.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - # Install Binaries - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - echo "running az login via identity" - az login --identity - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/oot-cred-provider.ps1 - - powershell C:/install-az-cli.ps1 - - powershell C:/replace-pr-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -624,20 +416,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity metadata: @@ -651,34 +429,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -834,254 +584,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml index df392e151ce..fb17eebdb28 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load-dra.yaml @@ -5,11 +5,7 @@ metadata: azuredisk-csi: "true" cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled storageclass: "true" - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -439,152 +435,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - #!/bin/bash - - echo "enabling containerd CDI plugin" - sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a\ enable_cdi = true' /etc/containerd/config.toml - systemctl restart containerd - owner: root:root - path: /tmp/containerd-config.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-"DynamicResourceAllocation=true"} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - bash -c /tmp/containerd-config.sh - - powershell c:/create-external-network.ps1 - - powershell C:/replace-pr-binaries.ps1 - - powershell C:/oot-cred-provider.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -619,48 +469,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -816,254 +624,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml index 482f4727487..3da04f1d541 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-load.yaml @@ -5,11 +5,7 @@ metadata: azuredisk-csi: "true" cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled storageclass: "true" - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -410,141 +406,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - - powershell C:/replace-pr-binaries.ps1 - - powershell C:/oot-cred-provider.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -579,48 +440,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -776,254 +595,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool-windows.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool-windows.yaml new file mode 100644 index 00000000000..8b049d46e3e --- /dev/null +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool-windows.yaml @@ -0,0 +1,1015 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + annotations: + controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: {} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login + done + else + echo "Using curl to download the binaries" + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + done + fi + systemctl restart kubelet + + # prepull images from gcr.io/k8s-staging-ci-images and retag it to + # registry.k8s.io so kubeadm can fetch correct images no matter what + declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for IMAGE in "$${IMAGES[@]}"; do + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}" + done + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + owner: root:root + path: /tmp/replace-k8s-binaries.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz + tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq + rm /tmp/yq_linux_amd64.tar.gz + + export KUBECONFIG=/etc/kubernetes/admin.conf + kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}" + systemctl stop kubelet + yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml + systemctl restart kubelet + owner: root:root + path: /tmp/replace-k8s-components.sh + permissions: "0744" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - bash -c /tmp/replace-k8s-components.sh + preKubeadmCommands: + - bash -c /tmp/replace-k8s-binaries.sh + - bash -c /tmp/oot-cred-provider.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + strategy: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + done + systemctl restart kubelet + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + owner: root:root + path: /tmp/replace-k8s-binaries.sh + permissions: "0744" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/replace-k8s-binaries.sh +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + location: ${AZURE_LOCATION} + template: + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" + if($$KUBE_GIT_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-pr-binaries.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + Write-Host "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell c:/create-external-network.ps1 + - powershell C:/replace-pr-binaries.ps1 + - powershell C:/oot-cred-provider.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: v1 +data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + priorityClassName: system-node-critical + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 8b049d46e3e..9bb3ff2c15e 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -4,10 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -400,141 +396,6 @@ spec: type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - - powershell C:/replace-pr-binaries.ps1 - - powershell C:/oot-cred-provider.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -569,48 +430,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -765,251 +584,3 @@ spec: imageName: "${CNM_IMAGE_NAME:-""}" imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" ---- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds-windows.yaml b/templates/test/dev/cluster-template-custom-builds-windows.yaml new file mode 100644 index 00000000000..b0621f1aabd --- /dev/null +++ b/templates/test/dev/cluster-template-custom-builds-windows.yaml @@ -0,0 +1,1307 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} + cni: calico + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + metrics-server: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + annotations: + controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + feature-gates: ${K8S_FEATURE_GATES:-""} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login + done + systemctl restart kubelet + + # prepull images from gcr.io/k8s-staging-ci-images and retag it to + # registry.k8s.io so kubeadm can fetch correct images no matter what + declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for IMAGE in "$${IMAGES[@]}"; do + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}" + done + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + owner: root:root + path: /tmp/replace-k8s-binaries.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz + tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq + rm /tmp/yq_linux_amd64.tar.gz + + export KUBECONFIG=/etc/kubernetes/admin.conf + kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}" + systemctl stop kubelet + yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml + systemctl restart kubelet + owner: root:root + path: /tmp/replace-k8s-components.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - bash -c /tmp/replace-k8s-components.sh + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/replace-k8s-binaries.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + additionalTags: + monitoring: virtualmachine + identity: UserAssigned + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-2204-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login + done + systemctl restart kubelet + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + owner: root:root + path: /tmp/replace-k8s-binaries.sh + permissions: "0744" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/replace-k8s-binaries.sh +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + selector: {} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + spec: + identity: UserAssigned + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" + if($$CONTAINERD_URL -ne ""){ + # Kubelet service depends on contianerd service so make a best effort attempt to stop it + Stop-Service kubelet -Force -ErrorAction SilentlyContinue + Stop-Service containerd -Force + echo "downloading containerd: $$CONTAINERD_URL" + curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" + tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 + + Start-Service containerd + } + + containerd.exe --version + containerd-shim-runhcs-v1.exe --version + path: C:/replace-containerd.ps1 + permissions: "0744" + - content: | + mkdir -Force c:/localdumps + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f + # Enable sftp so we can copy crash dump files during log collection of stfp + $sshd_config = "$env:ProgramData\ssh\sshd_config" + if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } + Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" + sc.exe stop sshd + sc.exe start sshd + path: C:/collect-hns-crashes.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + Write-Host "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" + - content: | + Write-Host "Installing Azure CLI" + $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "Version Azure CLI:" + az version + path: C:/install-az-cli.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" + if($$KUBE_GIT_VERSION -ne "") + { + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + # Install Binaries + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + echo "running az login via identity" + az login --identity + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-pr-binaries.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell C:/create-temp-folder.ps1 + - powershell C:/replace-containerd.ps1 + - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 + - powershell C:/install-az-cli.ps1 + - powershell C:/replace-pr-binaries.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + cluster.x-k8s.io/control-plane: "" + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico + namespace: default +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: Calico + calicoNetwork: + bgp: Disabled + windowsDataplane: HNS + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + typhaDeployment: + spec: + template: + spec: + # By default, typha tolerates all NoSchedule taints. This breaks + # scale-ins when it continuously gets scheduled onto an + # out-of-date Node that is being deleted. Tolerate only the + # NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + registry: capzcicommunity.azurecr.io + serviceCIDRs: + - 10.96.0.0/12 # must match cluster service CIDR (this is the default) + # Image and registry configuration for the tigera/operator pod + tigeraOperator: + image: tigera/operator + registry: capzcicommunity.azurecr.io + calicoctl: + image: capzcicommunity.azurecr.io/calico/ctl + # when kubernetesServiceEndpoint (required for windows) is added + # DNS configuration is needed to look up the api server name properly + # https://github.com/projectcalico/calico/issues/9536 + dnsConfig: + nameservers: + - 127.0.0.53 + options: + - name: edns0 + - name: trust-ad + kubernetesServiceEndpoint: + host: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + port: "{{ .Cluster.spec.controlPlaneEndpoint.port }}" + # By default, tigera tolerates all NoSchedule taints. This breaks upgrades + # when it continuously gets scheduled onto an out-of-date Node that is being + # deleted. Tolerate only the NoSchedule taints that are expected. + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: "${CCM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CCM:-""}" + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: "${CNM_IMAGE_NAME:-""}" + imageRepository: "${IMAGE_REGISTRY:-""}" + imageTag: "${IMAGE_TAG_CNM:-""}" +--- +apiVersion: v1 +data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + priorityClassName: system-node-critical + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: metrics-server-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + metrics-server: enabled + resources: + - kind: ConfigMap + name: metrics-server-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + metrics-server: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader + rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - emptyDir: {} + name: tmp-dir + --- + apiVersion: apiregistration.k8s.io/v1 + kind: APIService + metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io + spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: metrics-server-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index b0621f1aabd..aff755fce46 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -4,9 +4,6 @@ metadata: labels: cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled metrics-server: enabled name: ${CLUSTER_NAME} namespace: default @@ -61,8 +58,7 @@ spec: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - feature-gates: ${K8S_FEATURE_GATES:-""} + extraArgs: {} timeoutForControlPlane: 20m controllerManager: extraArgs: @@ -380,210 +376,6 @@ spec: - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - identity: UserAssigned - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Write-Host "Attempting to log in to Azure with managed identity" - az login --identity > $null 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "Logged in Azure with managed identity" - Write-Host "Use OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider --auth-mode login - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login - } else { - Write-Host "Using curl to download the OOT credential provider" - mkdir C:\var\lib\kubelet\credential-provider - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider - cp C:\var\lib\kubelet\credential-provider\acr-credential-provider C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe - curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml - } - path: C:/oot-cred-provider.ps1 - permissions: "0744" - - content: | - Write-Host "Installing Azure CLI" - $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - Write-Host "Version Azure CLI:" - az version - path: C:/install-az-cli.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" - if($$KUBE_GIT_VERSION -ne "") - { - # Need to add manually AZ to PATH as it is not added without a reset - Write-Host "Adding AZ to path:" - $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" - # Install Binaries - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - echo "running az login via identity" - az login --identity - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-pr-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider - image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/oot-cred-provider.ps1 - - powershell C:/install-az-cli.ps1 - - powershell C:/replace-pr-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: ${CLUSTER_NAME}-control-plane @@ -618,20 +410,6 @@ spec: timeout: 30s type: E2ENodeUnhealthy --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity metadata: @@ -645,34 +423,6 @@ spec: tenantID: ${AZURE_TENANT_ID} type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} --- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy metadata: @@ -828,254 +578,6 @@ spec: imageRepository: "${IMAGE_REGISTRY:-""}" imageTag: "${IMAGE_TAG_CNM:-""}" --- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: diff --git a/templates/test/dev/custom-builds-machine-pool-windows/kustomization.yaml b/templates/test/dev/custom-builds-machine-pool-windows/kustomization.yaml new file mode 100644 index 00000000000..3856563ecd1 --- /dev/null +++ b/templates/test/dev/custom-builds-machine-pool-windows/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../../../test/ci/prow-machine-pool-windows +patches: +- path: ../patches/control-plane-custom-builds.yaml +- path: patches/machine-pool-deployment-pr-version-windows.yaml +- path: patches/custom-builds.yaml +- path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfig + name: .*-mp-win + namespace: default + version: v1beta1 +configMapGenerator: +- behavior: merge + files: + - kube-proxy-patch=../../../test/ci/patches/windows-kubeproxy-ci.yaml + name: cni-${CLUSTER_NAME}-calico-windows +generatorOptions: + annotations: + note: generated + disableNameSuffixHash: true + labels: + type: generated +sortOptions: + order: fifo diff --git a/templates/test/dev/custom-builds-machine-pool-windows/patches/custom-builds.yaml b/templates/test/dev/custom-builds-machine-pool-windows/patches/custom-builds.yaml new file mode 100644 index 00000000000..8c9da017a84 --- /dev/null +++ b/templates/test/dev/custom-builds-machine-pool-windows/patches/custom-builds.yaml @@ -0,0 +1,86 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 +spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml + preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh + - bash -c /tmp/replace-k8s-binaries.sh + files: + - path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # Run the az login command with managed identity + if az login --identity > /dev/null 2>&1; then + echo "Logged in Azure with managed identity" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + else + echo "Using curl to download the OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + fi + - path: /tmp/replace-k8s-binaries.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + done + systemctl restart kubelet + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + - path: /etc/kubernetes/azure.json + owner: "root:root" + permissions: "0644" + contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 +spec: + template: + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi + sku: ubuntu-2204-gen1 + version: latest diff --git a/templates/test/dev/custom-builds-machine-pool-windows/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml b/templates/test/dev/custom-builds-machine-pool-windows/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml new file mode 100644 index 00000000000..542c444ce01 --- /dev/null +++ b/templates/test/dev/custom-builds-machine-pool-windows/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml @@ -0,0 +1,71 @@ +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" + if($$KUBE_GIT_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-pr-binaries.ps1 + permissions: "0744" +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + Write-Host "Attempting to log in to Azure with managed identity" + az login --identity > $null 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "Logged in Azure with managed identity" + Write-Host "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" -f C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe --auth-mode login + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" -f C:\var\lib\kubelet\credential-provider-config.yaml --auth-mode login + } else { + Write-Host "Using curl to download the OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml" --output C:\var\lib\kubelet\credential-provider-config.yaml + } + path: C:/oot-cred-provider.ps1 + permissions: "0744" +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/replace-pr-binaries.ps1 +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/dev/custom-builds-machine-pool-windows/patches/machine-pool-deployment-pr-version-windows.yaml b/templates/test/dev/custom-builds-machine-pool-windows/patches/machine-pool-deployment-pr-version-windows.yaml new file mode 100644 index 00000000000..a18713bbeb2 --- /dev/null +++ b/templates/test/dev/custom-builds-machine-pool-windows/patches/machine-pool-deployment-pr-version-windows.yaml @@ -0,0 +1,14 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win +spec: + template: + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi-windows + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest diff --git a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml index db2828771f8..c8ee9e6ac14 100644 --- a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml +++ b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml @@ -5,7 +5,6 @@ resources: - ../../../test/ci/prow-machine-pool patches: - path: ../patches/control-plane-custom-builds.yaml -- path: patches/machine-pool-deployment-pr-version-windows.yaml - path: patches/custom-builds.yaml - path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml target: @@ -13,23 +12,5 @@ patches: kind: KubeadmControlPlane name: .*-control-plane version: v1beta1 -- path: patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfig - name: .*-mp-win - namespace: default - version: v1beta1 -configMapGenerator: -- behavior: merge - files: - - kube-proxy-patch=../../../test/ci/patches/windows-kubeproxy-ci.yaml - name: cni-${CLUSTER_NAME}-calico-windows -generatorOptions: - annotations: - note: generated - disableNameSuffixHash: true - labels: - type: generated sortOptions: order: fifo diff --git a/templates/test/dev/custom-builds-windows/kustomization.yaml b/templates/test/dev/custom-builds-windows/kustomization.yaml new file mode 100644 index 00000000000..cea8c6e4096 --- /dev/null +++ b/templates/test/dev/custom-builds-windows/kustomization.yaml @@ -0,0 +1,69 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../../../test/ci/prow-windows +- ../../../addons/metrics-server/metrics-server-resource-set.yaml +patches: +- path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 +- path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + version: v1beta1 +- path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: patches/kubeadm-bootstrap.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + version: v1beta1 +- path: patches/kubeadm-controlplane-bootstrap.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: .*-control-plane + version: v1beta1 +- path: patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + version: v1beta1 +- path: patches/machine-deployment-pr-version.yaml +- path: patches/machine-deployment-pr-version-windows.yaml +- path: ../../../test/ci/patches/windows-feature-gates.yaml +- path: ../../../test/ci/patches/windows-containerd-labels.yaml +- path: ../../../test/ci/patches/windows-machine-deployment-replicas.yaml +- path: ../../../test/ci/patches/metrics-server-enabled-cluster.yaml +configMapGenerator: +- behavior: merge + files: + - kube-proxy-patch=../../../test/ci/patches/windows-kubeproxy-ci.yaml + name: cni-${CLUSTER_NAME}-calico-windows +- files: + - metrics-server=../../../addons/metrics-server/metrics-server.yaml + name: metrics-server-${CLUSTER_NAME} +generatorOptions: + annotations: + note: generated + disableNameSuffixHash: true + labels: + type: generated +sortOptions: + order: fifo diff --git a/templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml b/templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml new file mode 100644 index 00000000000..731a5ab3a45 --- /dev/null +++ b/templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml @@ -0,0 +1,58 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + Write-Host "Installing Azure CLI" + $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "Version Azure CLI:" + az version + path: C:/install-az-cli.ps1 + permissions: "0744" +- op: add + path: /spec/template/spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" + if($$KUBE_GIT_VERSION -ne "") + { + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + # Install Binaries + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + echo "running az login via identity" + az login --identity + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-pr-binaries.ps1 + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/install-az-cli.ps1 +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/replace-pr-binaries.ps1 diff --git a/templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap.yaml b/templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap.yaml new file mode 100644 index 00000000000..5f66b63b3fd --- /dev/null +++ b/templates/test/dev/custom-builds-windows/patches/kubeadm-bootstrap.yaml @@ -0,0 +1,29 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login + done + systemctl restart kubelet + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + path: /tmp/replace-k8s-binaries.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + bash -c /tmp/replace-k8s-binaries.sh diff --git a/templates/test/dev/custom-builds-windows/patches/kubeadm-controlplane-bootstrap.yaml b/templates/test/dev/custom-builds-windows/patches/kubeadm-controlplane-bootstrap.yaml new file mode 100644 index 00000000000..d876f4446ee --- /dev/null +++ b/templates/test/dev/custom-builds-windows/patches/kubeadm-controlplane-bootstrap.yaml @@ -0,0 +1,67 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + systemctl stop kubelet + declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity + for BINARY in "$${BINARIES[@]}"; do + echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login + done + systemctl restart kubelet + + # prepull images from gcr.io/k8s-staging-ci-images and retag it to + # registry.k8s.io so kubeadm can fetch correct images no matter what + declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for IMAGE in "$${IMAGES[@]}"; do + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}" + done + + echo "kubeadm version: $(kubeadm version -o=short)" + echo "kubectl version: $(kubectl version --client=true)" + echo "kubelet version: $(kubelet --version)" + path: /tmp/replace-k8s-binaries.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz + tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq + rm /tmp/yq_linux_amd64.tar.gz + + export KUBECONFIG=/etc/kubernetes/admin.conf + kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}" + systemctl stop kubelet + yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml + yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml + systemctl restart kubelet + path: /tmp/replace-k8s-components.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands/- + value: + bash -c /tmp/replace-k8s-binaries.sh +- op: add + path: /spec/kubeadmConfigSpec/postKubeadmCommands/- + value: + bash -c /tmp/replace-k8s-components.sh diff --git a/templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version-windows.yaml b/templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version-windows.yaml new file mode 100644 index 00000000000..c153570d2eb --- /dev/null +++ b/templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version-windows.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-win" +spec: + template: + spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi-windows + sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 + version: latest diff --git a/templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version.yaml b/templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version.yaml new file mode 100644 index 00000000000..60e367449c6 --- /dev/null +++ b/templates/test/dev/custom-builds-windows/patches/machine-deployment-pr-version.yaml @@ -0,0 +1,50 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi + sku: ubuntu-2204-gen1 + version: latest +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY} + image: + # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. + # latest binaries and images will get replaced to the desired version by the script above. + marketplace: + publisher: cncf-upstream + offer: capi + sku: ubuntu-2204-gen1 + version: latest +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + annotations: + controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: ci/${CI_VERSION} + verbosity: 5 diff --git a/templates/test/dev/custom-builds/kustomization.yaml b/templates/test/dev/custom-builds/kustomization.yaml index 56a412bd93a..7ce7e16d920 100644 --- a/templates/test/dev/custom-builds/kustomization.yaml +++ b/templates/test/dev/custom-builds/kustomization.yaml @@ -12,13 +12,6 @@ patches: name: .*-md-0 namespace: default version: v1beta1 -- path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - version: v1beta1 - path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml target: group: controlplane.cluster.x-k8s.io @@ -38,24 +31,9 @@ patches: kind: KubeadmControlPlane name: .*-control-plane version: v1beta1 -- path: patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - version: v1beta1 - path: patches/machine-deployment-pr-version.yaml -- path: patches/machine-deployment-pr-version-windows.yaml -- path: ../../../test/ci/patches/windows-feature-gates.yaml -- path: ../../../test/ci/patches/windows-containerd-labels.yaml -- path: ../../../test/ci/patches/windows-machine-deployment-replicas.yaml - path: ../../../test/ci/patches/metrics-server-enabled-cluster.yaml configMapGenerator: -- behavior: merge - files: - - kube-proxy-patch=../../../test/ci/patches/windows-kubeproxy-ci.yaml - name: cni-${CLUSTER_NAME}-calico-windows - files: - metrics-server=../../../addons/metrics-server/metrics-server.yaml name: metrics-server-${CLUSTER_NAME} diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index dc7e94bb421..e0cdd413a30 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -695,7 +695,7 @@ func resolveKubernetesVersions(config *clusterctl.E2EConfig) { // Check if Windows testing is explicitly disabled via TEST_WINDOWS environment variable testWindows := os.Getenv("TEST_WINDOWS") - windowsRequired := testWindows != "false" + windowsRequired := testWindows == "true" if windowsRequired { windowsVersions := getVersionsInCommunityGallery(context.TODO(), os.Getenv(AzureLocation), capiCommunityGallery, "capi-win-2019-containerd") From ccfd498feb4df11612cbfa9f469bac5dc5daf59b Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 14 Oct 2025 14:41:06 -0700 Subject: [PATCH 2/6] omit Windows pool from ClusterClass test scenario --- test/e2e/azure_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 466f5d44b4e..c7ca2d7e72c 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -1046,16 +1046,13 @@ var _ = Describe("Workload cluster creation", func() { }) Context("Creating clusters using clusterclass [OPTIONAL]", func() { - It("with a single control plane node, one linux worker node, and one windows worker node", func() { + It("with a single control plane node and one linux worker node", func() { // Use ci-default as the clusterclass name so test infra can find the clusterclass template Expect(os.Setenv("CLUSTER_CLASS_NAME", "ci-default")).To(Succeed()) // Use "cc" as spec name because NAT gateway pip name exceeds limit. clusterName = getClusterName(clusterNamePrefix, "cc") - // Opt into using windows with prow template - Expect(os.Setenv("WINDOWS_WORKER_MACHINE_COUNT", "1")).To(Succeed()) - // Create a cluster using the cluster class created above clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, From c176acc9fb47cbb504de3c788698b4ee3e06d1d3 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 14 Oct 2025 14:41:53 -0700 Subject: [PATCH 3/6] use new machine-pool-windows template for VMSS test --- test/e2e/azure_test.go | 2 +- test/e2e/config/azure-dev.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index c7ca2d7e72c..7f872799876 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -518,7 +518,7 @@ var _ = Describe("Workload cluster creation", func() { clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, - withFlavor("machine-pool"), + withFlavor("machine-pool-windows"), withNamespace(namespace.Name), withClusterName(clusterName), withControlPlaneMachineCount(1), diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 2e968fcf673..ac63c78ee56 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -130,6 +130,8 @@ providers: - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-machine-and-machine-pool.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool.yaml" targetName: "cluster-template-machine-pool.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml" + targetName: "cluster-template-machine-pool-windows.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ipv6.yaml" targetName: "cluster-template-ipv6.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml" From 841af616f599fa5139bb50077c3a283447959b2d Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 15 Oct 2025 10:05:25 -0700 Subject: [PATCH 4/6] add a separate Windows test --- test/e2e/azure_test.go | 67 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 7f872799876..558015c2d28 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -510,9 +510,74 @@ var _ = Describe("Workload cluster creation", func() { }) Context("Creating a VMSS cluster [REQUIRED]", func() { - It("with a single control plane node and an AzureMachinePool with 2 Linux and 2 Windows worker nodes", func() { + It("with a single control plane node and an AzureMachinePool with 2 Linux nodes", func() { clusterName = getClusterName(clusterNamePrefix, "vmss") + clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( + specName, + withFlavor("machine-pool"), + withNamespace(namespace.Name), + withClusterName(clusterName), + withControlPlaneMachineCount(1), + withWorkerMachineCount(2), + withMachineDeploymentInterval(specName, ""), + withControlPlaneInterval(specName, "wait-control-plane"), + withMachinePoolInterval(specName, "wait-machine-pool-nodes"), + withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + }), + withPostMachinesProvisioned(func() { + EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { + return DaemonsetsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }), + ), result) + + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + + By("Creating an accessible load balancer", func() { + AzureLBSpec(ctx, func() AzureLBSpecInput { + return AzureLBSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + + By("Creating an accessible load balancer for windows", func() { + AzureLBSpec(ctx, func() AzureLBSpecInput { + return AzureLBSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + Windows: true, + } + }) + }) + + By("PASSED!") + }) + }) + + Context("Creating a VMSS cluster with Windows [WINDOWS]", func() { + It("with a single control plane node and an AzureMachinePool with 2 Linux and 2 Windows worker nodes", func() { + clusterName = getClusterName(clusterNamePrefix, "vmss-windows") + // Opt into using windows with prow template Expect(os.Setenv("WINDOWS_WORKER_MACHINE_COUNT", "2")).To(Succeed()) From da4f547025af6a0492033ae4a82c39b944a1d5e8 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 15 Oct 2025 11:39:08 -0700 Subject: [PATCH 5/6] non-Windows test scenario should not test Windows LB --- test/e2e/azure_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 558015c2d28..58433fa7d7f 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -558,18 +558,6 @@ var _ = Describe("Workload cluster creation", func() { }) }) - By("Creating an accessible load balancer for windows", func() { - AzureLBSpec(ctx, func() AzureLBSpecInput { - return AzureLBSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - SkipCleanup: skipCleanup, - Windows: true, - } - }) - }) - By("PASSED!") }) }) From 414e7992afc88ad31b9eadf29df5b444088b6538 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 15 Oct 2025 16:46:38 -0700 Subject: [PATCH 6/6] =?UTF-8?q?use=20shorter=20=E2=80=9Cwin=E2=80=9D=20nam?= =?UTF-8?q?e=20to=20avoid=20API=20issues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/azure_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 58433fa7d7f..97eff3cfebf 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -564,7 +564,7 @@ var _ = Describe("Workload cluster creation", func() { Context("Creating a VMSS cluster with Windows [WINDOWS]", func() { It("with a single control plane node and an AzureMachinePool with 2 Linux and 2 Windows worker nodes", func() { - clusterName = getClusterName(clusterNamePrefix, "vmss-windows") + clusterName = getClusterName(clusterNamePrefix, "vmss-win") // Opt into using windows with prow template Expect(os.Setenv("WINDOWS_WORKER_MACHINE_COUNT", "2")).To(Succeed())