upgrade-test: add 2 upgrades test

- Do client-side layering tests on old build, then upgrade
rpm-ostree to new version, then upgrade to another new update.
- Start old build, upgrade rpm-ostree to new version, do
client-side layering tests, then upgrade to another new update.

.cci.jenkinsfile

@@ -77,4 +77,13 @@ cosaPod(runAsUser: 0, memory: "${mem}Mi", cpu: "${nhosts}") {
       archiveArtifacts allowEmptyArchive: true, artifacts: 'vmcheck-logs.tar.xz'
     }
   }
+  stage("kola-upgrades") {
+    unstash 'rpms'
+    shwrap("""
+      coreos-installer download -p qemu -f qcow2.xz -s testing --decompress
+      make -C tests/kolainst upgrades-install
+      mv rpm-ostree-[0-9]*.rpm rpm-ostree-libs-[0-9]*.rpm /usr/lib/coreos-assembler/tests/kola/rpm-ostree/upgrades/data/      
+    """)
+    kola(cosaDir: "${env.WORKSPACE}", extraArgs: "--qemu-image ./fedora-coreos*.qcow2 ext.rpm-ostree.upgrades.*", parallel: 2)
+  }
 }

tests/common/libtest.sh

@@ -539,3 +539,56 @@ rpmostree_busctl_call_os() {
   busctl call org.projectatomic.rpmostree1 $ospath \
     org.projectatomic.rpmostree1.OS "$@"
 }
+
+# This is used by kolainst/upgrades
+# Do client-side layering tests includes:
+# kernel override, initramfs args, initramfs-etc, layering, overrides
+client_side_layering_tests()
+{
+  # override kernel
+  # copy test code from test-override-kernel.sh
+  current=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+  rpm-ostree db list "${current}" > current-dblist.txt
+  if grep -qF $kernel_release current-dblist.txt; then
+    echo "Should not find $kernel_release in current deployment"
+    exit 1
+  fi
+
+  grep -E '^ kernel-[0-9]' current-dblist.txt  | sed -e 's,^ *,,' > orig-kernel.txt
+  test "$(wc -l < orig-kernel.txt)" == "1"
+  orig_kernel=$(cat orig-kernel.txt)
+
+  rpm-ostree override replace $koji_kernel_url
+
+  rpm-ostree kargs --append foo=bar
+
+  touch /etc/foobar.conf
+  rpm-ostree initramfs --enable --arg=-I --arg=/etc/foobar.conf
+
+  rpm-ostree override remove moby-engine
+  rpm-ostree install --cache-only ${KOLA_EXT_DATA}/rpm-repos/0/packages/x86_64/foo-1.2-3.x86_64.rpm
+}
+
+# Check client-side layering tests results
+client_side_layering_checking()
+{
+  rpm-ostree status
+
+  test "$(uname -r)" == $kernel_release
+
+  cat /proc/cmdline > cmdlinekargs.txt
+  if ! grep foo=bar cmdlinekargs.txt; then
+    echo "can not find kernel parameter foo=bar"
+    exit 1
+  fi
+  lsinitrd "/usr/lib/modules/$(uname -r)/initramfs.img" > lsinitrd.txt
+  if ! grep etc/foobar.conf lsinitrd.txt; then
+    echo "can not find file expected to be included in initramfs.img"
+    exit 1
+  fi
+  if rpm -q moby-engine 2>/dev/null; then
+    echo "found package expected to be removed"
+    exit 1
+  fi
+  rpm -q foo
+}

tests/kolainst/Makefile

@@ -22,3 +22,9 @@ install: all
 localinstall: all
 	rm -rf ../kola
 	make install KOLA_TESTDIR=../kola
+
+upgrades-install: all
+	install -d -m 0755 $(KOLA_TESTDIR)
+	rsync -prlv ./upgrades $(KOLA_TESTDIR)/
+	rsync -prlv ../common/*.sh $(KOLA_TESTDIR)/upgrades/data/
+	rsync -prlv rpm-repos/ $(KOLA_TESTDIR)/upgrades/data/rpm-repos/

tests/kolainst/upgrades/client-layering

@@ -0,0 +1,86 @@
+#!/bin/bash
+## kola:
+##   # This test reboots a lot, generates container images, etc.
+##   timeoutMin: 40
+##   minMemory: 2048
+##   tags: needs-internet
+##   description: Do client-side layering tests on old build, then upgrade 
+##     rpm-ostree to new version, then upgrade to another new update.
+
+# Start old build, do client-side layering tests, reboot;
+# upgrade to new rpm-ostree, reboot;
+# exported to a container image from current commit, then rebase to it, reboot;
+# then upgrade to another update, reboot.
+
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libtest.sh
+
+. /etc/os-release
+case $VERSION_ID in
+  39) kernel_release=6.5.6-300.fc39.x86_64
+    koji_kernel_url="https://koji.fedoraproject.org/koji/buildinfo?buildID=2302642"
+  ;;
+  *) echo "Unsupported Fedora version: $VERSION_ID"
+    exit 1
+  ;;
+esac
+
+cd "$(mktemp -d)"
+
+image=containers-storage:localhost/fcos
+image_pull=ostree-unverified-image:$image
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    rpm-ostree --version
+    systemctl mask --now zincati
+
+    client_side_layering_tests
+    /tmp/autopkgtest-reboot 1
+    ;;
+  "1")
+    client_side_layering_checking
+    echo "ok client-side layering tests"
+
+    # update rpm-ostree
+    rpm-ostree override replace ${KOLA_EXT_DATA}/rpm-ostree*
+    /tmp/autopkgtest-reboot 2
+    ;;
+  "2")
+    client_side_layering_checking
+    echo "ok upgrade to new rpm-ostree"
+
+    # remove foo as workaround, see https://github.com/coreos/rpm-ostree/issues/4805
+    rpm-ostree uninstall foo
+    checksum=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+    ostree container encapsulate --repo=/ostree/repo ${checksum} ${image}
+
+    # rebase to container image
+    rpm-ostree rebase ${image_pull}
+    /tmp/autopkgtest-reboot 3    
+    ;;
+  "3")
+    rpm-ostree status
+
+    booted_image=$(rpm-ostree status --json | jq -r '.deployments[0]["container-image-reference"]')
+    test "${image_pull}" = "$booted_image"
+    echo "ok rebase container image"
+
+    cat > Containerfile << EOF
+FROM localhost/fcos
+RUN rpm-ostree install vim-filesystem && rpm-ostree cleanup -m
+EOF
+
+    podman build --net=host -t localhost/fcos-derived --squash .
+    rpm-ostree rebase "${image_pull}-derived"
+    /tmp/autopkgtest-reboot 4
+    ;;
+  "4")
+    rpm-ostree status
+    rpm -q vim-filesystem
+    echo "ok upgrade to another container image"
+    ;;
+  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
+esac

tests/kolainst/upgrades/upgrade-rpmostree

@@ -0,0 +1,86 @@
+#!/bin/bash
+## kola:
+##   # This test reboots a lot, generates container images, etc.
+##   timeoutMin: 40
+##   minMemory: 2048
+##   tags: needs-internet
+##   description: Start old build, upgrade rpm-ostree to new version,
+##     do client-side layering tests, then upgrade to another new update.
+
+# start old build, upgrade to new rpm-ostree, reboot;
+# do client-side layering tests, reboot;
+# upgrade to container image, reboot;
+# then upgrade to another update, reboot.
+
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libtest.sh
+
+. /etc/os-release
+case $VERSION_ID in
+  39) kernel_release=6.5.6-300.fc39.x86_64
+    koji_kernel_url="https://koji.fedoraproject.org/koji/buildinfo?buildID=2302642"
+  ;;
+  *) echo "Unsupported Fedora version: $VERSION_ID"
+    exit 1
+  ;;
+esac
+
+cd "$(mktemp -d)"
+
+image=containers-storage:localhost/fcos
+image_pull=ostree-unverified-image:$image
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    rpm-ostree --version
+    systemctl mask --now zincati
+
+    # update rpm-ostree
+    rpm-ostree override replace ${KOLA_EXT_DATA}/rpm-ostree*
+    /tmp/autopkgtest-reboot 1
+    ;;
+  "1")
+    rpm-ostree status
+    echo "ok upgrade rpm-ostree"
+
+    client_side_layering_tests
+    /tmp/autopkgtest-reboot 2
+    ;;
+  "2")
+    client_side_layering_checking
+    echo "ok client-side layering tests with new rpm-ostree"
+
+    # remove foo as workaround, see https://github.com/coreos/rpm-ostree/issues/4805
+    rpm-ostree uninstall foo
+    checksum=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+    ostree container encapsulate --repo=/ostree/repo ${checksum} ${image}
+
+    # rebase to container image
+    rpm-ostree rebase ${image_pull}
+    /tmp/autopkgtest-reboot 3    
+    ;;
+  "3")
+    rpm-ostree status
+
+    booted_image=$(rpm-ostree status --json | jq -r '.deployments[0]["container-image-reference"]')
+    test "${image_pull}" = "$booted_image"
+    echo "ok rebase container image"
+
+    cat > Containerfile << EOF
+FROM localhost/fcos
+RUN rpm-ostree install vim-filesystem && rpm-ostree cleanup -m
+EOF
+
+    podman build --net=host -t localhost/fcos-derived --squash .
+    rpm-ostree rebase "${image_pull}-derived"
+    /tmp/autopkgtest-reboot 4
+    ;;
+  "4")
+    rpm-ostree status
+    rpm -q vim-filesystem
+    echo "ok upgrade to another container image"
+    ;;
+  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
+esac