diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d5feac5bf..051b59e4d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -88,7 +88,9 @@ jobs: overwrite: true e2e-tests: + continue-on-error: true strategy: + fail-fast: false matrix: kubernetes: [ "v1.23.17", "v1.24.15", "v1.25.11", "v1.26.6" ] replicas: ["1"] diff --git a/.obs/chartfile/crds/templates/crds.yaml b/.obs/chartfile/crds/templates/crds.yaml index b97c41c04..4bdee9e65 100644 --- a/.obs/chartfile/crds/templates/crds.yaml +++ b/.obs/chartfile/crds/templates/crds.yaml @@ -2,10 +2,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: machineinventories.elemental.cattle.io spec: @@ -198,10 +202,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: machineinventoryselectors.elemental.cattle.io spec: @@ -405,10 +413,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: machineinventoryselectortemplates.elemental.cattle.io spec: @@ -640,10 +652,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: machineregistrations.elemental.cattle.io spec: @@ -964,10 +980,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: managedosimages.elemental.cattle.io spec: @@ -2502,10 +2522,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: managedosversionchannels.elemental.cattle.io spec: @@ -2998,10 +3022,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: managedosversions.elemental.cattle.io spec: @@ -3404,10 +3432,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: metadata.elemental.cattle.io spec: @@ -3461,10 +3493,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/part-of: Elemental Operator + app.kubernetes.io/version: '{{ .Chart.Version }}' controller-gen.kubebuilder.io/version: v0.14.0 labels: cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' release-name: '{{ .Release.Name }}' name: seedimages.elemental.cattle.io spec: diff --git a/.obs/chartfile/operator/templates/channel-dev.yaml b/.obs/chartfile/operator/templates/channel-dev.yaml new file mode 100644 index 000000000..da6678403 --- /dev/null +++ b/.obs/chartfile/operator/templates/channel-dev.yaml @@ -0,0 +1,13 @@ +# Unstable channel for testing isv:Rancher:Elemental OBS projects +# it is only rendered if the registryUrl value includes a known OBS project reference +{{ if and (hasPrefix "registry.opensuse.org" .Values.registryUrl) (contains "isv/rancher/elemental" .Values.registryUrl) }} +apiVersion: elemental.cattle.io/v1beta1 +kind: ManagedOSVersionChannel +metadata: + name: unstable-testing-channel + namespace: fleet-default +spec: + options: + image: {{ .Values.registryUrl }}/rancher/elemental-unstable-channel:latest + type: custom +{{ end }} diff --git a/.obs/chartfile/operator/templates/channel.yaml b/.obs/chartfile/operator/templates/channel.yaml deleted file mode 100644 index ccb6e4d3c..000000000 --- a/.obs/chartfile/operator/templates/channel.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{ if and .Values.channel .Values.channel.image .Values.channel.tag }} -apiVersion: elemental.cattle.io/v1beta1 -kind: ManagedOSVersionChannel -metadata: - name: elemental-channel - namespace: fleet-default -spec: - options: - image: {{ .Values.channel.image }}:{{ .Values.channel.tag }} - type: custom -{{ end }} diff --git a/.obs/chartfile/operator/templates/channels.yaml b/.obs/chartfile/operator/templates/channels.yaml new file mode 100644 index 000000000..c180d36d0 --- /dev/null +++ b/.obs/chartfile/operator/templates/channels.yaml @@ -0,0 +1,30 @@ +{{ $defChannelName := "" }} +{{ if and .Values.channel .Values.channel.image .Values.channel.tag .Values.channel.name }} +{{ $defChannelName := .Values.channel.name }} +apiVersion: elemental.cattle.io/v1beta1 +kind: ManagedOSVersionChannel +metadata: + name: {{ .Values.channel.name }} + namespace: fleet-default +spec: + options: + image: {{ .Values.channel.image }}:{{ .Values.channel.tag }} + type: custom +{{ end }} + +# Keep pre-existing channels managed by Helm if they do not match with the current default +# this way if an upgrade introduces a new channel any pre-existing channel managed by Helm is not deleted +{{ range $index, $channel := (lookup "elemental.cattle.io/v1beta1" "ManagedOSVersionChannel" "fleet-default" "").items }} + {{ if and (eq (index $channel.metadata.labels "app.kubernetes.io/managed-by") "Helm") (ne $channel.metadata.name $defChannelName) }} +--- +apiVersion: elemental.cattle.io/v1beta1 +kind: ManagedOSVersionChannel +metadata: + name: {{ $channel.metadata.name }} + namespace: fleet-default +spec: + options: + image: {{ $channel.spec.options.image }} + type: custom + {{ end }} +{{ end }} diff --git a/.obs/chartfile/operator/templates/validate-install-crd.yaml b/.obs/chartfile/operator/templates/validate-install-crd.yaml index 45008251a..a42cdbeb8 100644 --- a/.obs/chartfile/operator/templates/validate-install-crd.yaml +++ b/.obs/chartfile/operator/templates/validate-install-crd.yaml @@ -22,5 +22,9 @@ {{- if eq $crdrelease $.Release.Name -}} {{- required "Elemental CRDs should be moved to the new elemental-operator-crds chart before upgrading this operator." "" -}} {{- end -}} + {{- $crdversion := index $crdobj.metadata.annotations "app.kubernetes.io/version" -}} + {{- if or (not $crdversion) (ne $crdversion $.Chart.Version) -}} + {{- required "Elemental Operator CRDs chart version must match the version of this chart. Please install the corresponding CRD chart before." "" -}} + {{- end -}} {{- end -}} {{- end -}} diff --git a/.obs/chartfile/operator/values.yaml b/.obs/chartfile/operator/values.yaml index 5a06a0d9a..b19c002fe 100644 --- a/.obs/chartfile/operator/values.yaml +++ b/.obs/chartfile/operator/values.yaml @@ -10,8 +10,9 @@ seedImage: imagePullPolicy: IfNotPresent channel: - image: "%%IMG_REPO%%/rancher/elemental-channel" - tag: "%VERSION%" + name: "sl-micro-6.0-baremetal-channel" + image: "%%IMG_REPO%%/rancher/elemental-channel/sl-micro" + tag: "6.0-baremetal" # number of operator replicas to deploy replicas: 1 diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 2a04895e4..9b41da0af 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -17,3 +17,9 @@ commonLabels: release-name: '{{ .Release.Name }}' cluster.x-k8s.io/provider: infrastructure-elemental cluster.x-k8s.io/v1beta1: v1beta1 + helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' + +commonAnnotations: + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/version: '{{ .Chart.Version }}' + app.kubernetes.io/part-of: 'Elemental Operator' diff --git a/controllers/managedosversionchannel_controller_test.go b/controllers/managedosversionchannel_controller_test.go index 1f6b82d75..5b305b5de 100644 --- a/controllers/managedosversionchannel_controller_test.go +++ b/controllers/managedosversionchannel_controller_test.go @@ -212,8 +212,7 @@ var _ = Describe("reconcile managed os version channel", func() { // Re-sync is triggered to interval res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: name}) Expect(err).ToNot(HaveOccurred()) - Expect(res.RequeueAfter).To(BeNumerically("<", 1*time.Minute)) - Expect(res.RequeueAfter).To(BeNumerically(">", 59*time.Second)) + Expect(res.RequeueAfter).To(BeNumerically("~", 59*time.Second, 1*time.Minute)) }) It("should reconcile managed os version channel object without a type", func() { diff --git a/scripts/elemental-airgap.sh b/scripts/elemental-airgap.sh index ca47c23fc..fb96f6671 100755 --- a/scripts/elemental-airgap.sh +++ b/scripts/elemental-airgap.sh @@ -26,25 +26,28 @@ HAULER_REG_DUMMY_FILE="hauler-registry.txt" : ${CHANNEL_ONLY:="false"} : ${CHANNEL_IMAGE_NAME:="\$CHANNEL_IMAGE_NAME"} : ${SKIP_ARCHIVE_CREATION:="false"} +: ${ALL_CHANNELS:="false"} print_help() { cat <<- EOF Usage: $0 [OPTION] -r LOCAL_REGISTRY ELEMENTAL_OPERATOR_CHART + [-ac|-all-channels] add all defined ManagedOSVersionChannel. [-c|--crds-chart] Elemental CRDS chart (if URL, will be downloaded). - [-co|--channel-only] just extract and rebuild the ManagedOSVersionChannel container image - [-cv|--chart-version] Specify the chart version (only used if passing chart as URLs). + [-co|--channel-only] just extract and rebuild the ManagedOSVersionChannel container image. + [-cv|--chart-version] specify the chart version (only used if passing chart as URLs). [-d|--debug] enable debug output on screen. [-i|--images path] tar.gz gernerated by docker save. - [-h|--help] Usage message. - [-ha|--hauler] Use hauler to generate the archive (an "Haul") of the Elemental "Collection". + [-h|--help] usage message. + [-ha|--hauler] use hauler to generate the archive (an "Haul") of the Elemental "Collection". [-l|--image-list path] generated text file with the list of saved images (one image per line). [-r|--local-registry] registry where to load the images to (used in the next steps). - [-sa|--skip-archive] put the list of images in the $CONTAINER_IMAGES_FILE but skip $CONTAINER_IMAGES_ARCHIVE creation + [-sa|--skip-archive] put the list of images in the $CONTAINER_IMAGES_FILE but skip $CONTAINER_IMAGES_ARCHIVE creation. ELEMENTAL_OPERATOR_CHART could be either a chart tgz file or an url (in that case will be downloaded first) it could even be 'dev', 'staging' or 'stable' to allow automatic download of the charts. Parameters could also be set passing env vars: + ALL_CHANNELS (-ac) : $ALL_CHANNELS CONTAINER_IMAGES_NAME : $CONTAINER_IMAGES_NAME CONTAINER_IMAGES_FILE (-l) : $CONTAINER_IMAGES_FILE CONTAINER_IMAGES_ARCHIVE (-i) : $CONTAINER_IMAGES_ARCHIVE @@ -127,6 +130,10 @@ parse_parameters() { fi shift ;; + -ac|--all-channels) + ALL_CHANNELS=true + shift + ;; *) [[ -n "$CHART_NAME_OPERATOR" ]] && exit_error "unrecognized command: $1" CHART_NAME_OPERATOR="$1" @@ -352,6 +359,7 @@ build_os_channel() { local channel_img local channel_tag local channel_repo + local channel_list log_info "Creating OS channel" # name of the new channel container image we are going to create @@ -384,28 +392,54 @@ build_os_channel() { fi get_chart_val channel_tag "channel.tag" + channel_list+="${channel_img}:${channel_tag} " + + # we can have OS channels added in templates, so we have to sync them if needed + if [[ "$ALL_CHANNELS" == "true" ]]; then + # get all ManagedOSVersionChannel, so the already extracted one is in, we can overwrite channel_list + channel_list=$(helm template $CHART_NAME_OPERATOR \ + | yq 'select(.kind=="ManagedOSVersionChannel") .spec.options.image' \ + | sed -e s/\"//g -e /^---$/d 2>&1) + fi - if [[ -z "$channel_img" || -z "$channel_tag" ]]; then + if [[ -z "${channel_list// /}" ]]; then log_info "\nWARNING: channel image not found: you will need to provide your own Elemental OS images\n" return 0 fi - log_info "Found channel image: ${channel_img}:${channel_tag}" TEMPDIR=$(mktemp -d) log_debug "build channel image in $TEMPDIR" pushd $TEMPDIR > /dev/null - # extract the channel.json - if ! docker run --entrypoint busybox ${channel_img}:${channel_tag} cat channel.json > channel.json; then - exit_error "cannot extract OS images" + + # loop on the channel list + for channel in ${channel_list}; do + channel_img=${channel%:*} + channel_tag=${channel#*:} + + log_info "Found channel image: ${channel_img}:${channel_tag}" + + # extract the channel.json + if ! docker run --entrypoint busybox ${channel_img}:${channel_tag} cat channel.json > channel_${channel_img//\//_}.json; then + exit_error "cannot extract OS images" + fi + done + + # Merge all channel_*.json files + if ! jq -s add channel_*.json > channel.json; then + exit_error "cannot merge channel json files" fi # write the new channel and identify OS images to save local new_channel="" - for i in $(seq 0 20); do + local -i index=0 + while true; do local item item_type item_image item_name item_url_field - item=$(jq .[$i] channel.json) + item=$(jq .[$index] channel.json) [[ "$item" == "null" ]] && break + # increment index + index+=1 + get_json_val item_name "$item" ".metadata.name" get_json_val item_type "$item" ".spec.type" get_json_val item_display "$item" ".spec.metadata.displayName" diff --git a/tests/e2e/downgrade_test.go b/tests/e2e/downgrade_test.go new file mode 100644 index 000000000..b93f66117 --- /dev/null +++ b/tests/e2e/downgrade_test.go @@ -0,0 +1,103 @@ +/* +Copyright © 2022 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" + "github.com/rancher/elemental-operator/tests/e2e/config" + "sigs.k8s.io/controller-runtime/pkg/client" + + kubectl "github.com/rancher-sandbox/ele-testhelpers/kubectl" +) + +const stableCRDSChart = "oci://registry.suse.com/rancher/elemental-operator-crds-chart" +const stableChart = "oci://registry.suse.com/rancher/elemental-operator-chart" + +var _ = Describe("Elemental Operator downgrade/upgrade test", func() { + var k *kubectl.Kubectl + var downgradeCfg config.E2EConfig + var channelName string + var managedByLabel map[string]string + + It("downgrades to latest stable version", func() { + k = kubectl.New() + downgradeCfg = *e2eCfg + downgradeCfg.CRDsChart = stableCRDSChart + downgradeCfg.Chart = stableChart + channelName = "" + managedByLabel = map[string]string{ + "app.kubernetes.io/managed-by": "Helm", + } + + Expect(isOperatorInstalled(k)).To(BeTrue()) + + By("Uninstall Elemental Operator first before downgrading CRDs chart", func() { + Expect(kubectl.RunHelmBinaryWithCustomErr( + "uninstall", operatorName, + "--namespace", operatorNamespace, + "--wait", + )).To(Succeed()) + }) + + By("Install the new Elemental Operator", func() { + deployOperator(k, &downgradeCfg) + }) + + By("Check it gets a default channel and syncs ManagedOSVersions", func() { + Eventually(func() string { + channels := &elementalv1.ManagedOSVersionChannelList{} + err := cl.List(ctx, channels, client.InNamespace(fleetDefaultNamespace), client.MatchingLabels(managedByLabel)) + if err == nil { + // After uninstalling Operator and reinstalling it there should be only a single channel managed by Helm + Expect(len(channels.Items)).To(Equal(1)) + channelName = channels.Items[0].Name + } + return channelName + }, 10*time.Second, 1*time.Second).ShouldNot(BeEmpty()) + + Eventually(func() int { + mOSes := &elementalv1.ManagedOSVersionList{} + err := cl.List(ctx, mOSes, client.InNamespace(fleetDefaultNamespace), client.MatchingLabels(map[string]string{ + elementalv1.ElementalManagedOSVersionChannelLabel: channelName, + })) + if err == nil { + return len(mOSes.Items) + } + return 0 + }, 2*time.Minute, 2*time.Second).Should(BeNumerically(">", 0)) + }) + + By("Upgrading Elemental Operator to testing version", func() { + deployOperator(k, e2eCfg) + }) + + By("Check we still keep the previous default channel managed by Helm", func() { + Eventually(func() error { + ch := &elementalv1.ManagedOSVersionChannel{} + err := cl.Get(ctx, client.ObjectKey{ + Name: channelName, + Namespace: fleetDefaultNamespace, + }, ch) + return err + }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + }) + }) +}) diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index 36f6625da..6d3a0c5ac 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -380,10 +380,14 @@ func getElementalOperatorLogs() { )).To(Succeed()) for _, pod := range podList.Items { - for _, container := range pod.Spec.Containers { - output, err := kubectl.Run("logs", pod.Name, "-c", container.Name, "-n", pod.Namespace) - Expect(err).ToNot(HaveOccurred()) - Expect(os.WriteFile(filepath.Join(e2eCfg.ArtifactsDir, pod.Name+"-"+container.Name+".log"), []byte(output), 0644)).To(Succeed()) + if pod.Status.Phase == corev1.PodRunning { + for _, cStatus := range pod.Status.ContainerStatuses { + if cStatus.Started != nil && *cStatus.Started { + output, err := kubectl.Run("logs", pod.Name, "-c", cStatus.Name, "-n", pod.Namespace) + Expect(err).ToNot(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(e2eCfg.ArtifactsDir, pod.Name+"-"+cStatus.Name+".log"), []byte(output), 0644)).To(Succeed()) + } + } } } }