WIP install: new test to check against reference files

Tests for generation of the reference files, and for checking against them.

FIXME: needs reference files for releases

lib/host.py

@@ -600,3 +600,8 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def firmware_type(self):
+        retcode = self.ssh(['test', '-d', '/sys/firmware/efi/'],
+                           check=False, simple_output=False).returncode
+        return "uefi" if retcode == 0 else "bios"

scripts/runtests-get-fsdiff-ref

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+# extract reference dumps from install
+
+# needs at least --hosts=$NEST
+# needs runtest-install output cache
+
+VERSION="83nightly"
+
+. $(dirname $0)/lib.sh
+
+GITREV=$(git rev-parse HEAD)
+
+declare -A FSDIFFCONFS=()
+for conf in "${TESTCONFS[@]}"; do
+    IFS=- read fw sr < <(echo "$conf")
+    [ "$media" = "iso" -a "$sr" = "ext" ] || continue
+    FSDIFFCONFS[$fw-$VERSION]=$conf
+done
+
+for fsdiffconf in "${!FSDIFFCONFS[@]}"; do # "in FSDIFFCONFS.keys"
+    IFS=- read fw version < <(echo "$conf")
+    # get ref data for install
+    run_pytest \
+        --hosts="cache://install.test::Nested::boot_inst[$fw-$version-host1-iso-ext]-vm1-$GITREV" \
+        --log-file=test-fsdiffref-$conf.log \
+        tests/fs-diff/test_fsdiff_ref.py::test_fsdiff_mkref
+done
+
+report_failures

scripts/runtests-postinstall.sh

@@ -18,6 +18,12 @@ for conf in "${TESTCONFS[@]}"; do
         --runner "./jobs.py run postinstall cache://${host1},cache://${host2}" \
         --log-file=test-postinstall-$conf.log \
         "$@"
+
+    run_pytest \
+        --hosts="cache://install.test::Nested::boot_inst[$fw-83nightly-host1-iso-ext]-vm1-$GITREV" \
+        --log-file=test-fsdiff-$conf.log \
+        "$@" \
+        tests/fs-diff/test_fsdiff_ref.py::test_fsdiff_against_ref
 done
 
 report_failures

scripts/runtests-postupgrade.sh

@@ -37,6 +37,13 @@ for conf in "${REFCONFS[@]}"; do
             # FIXME postinstall-intrapool-migrate (with TLS) needs a 2-host pool
             ;;
     esac
+
+    # FIXME: check fsdiff with TLS activated, since that's what the 8.3+ ref have
+
+    run_pytest \
+        --hosts="cache://${host1}" \
+        --log-file=test-fsdiff-$conf.log \
+        tests/fs-diff/test_fsdiff_ref.py::test_fsdiff_against_ref
 done
 
 report_failures

tests/fs-diff/test_fsdiff_ref.py

@@ -0,0 +1,36 @@
+import logging
+import os
+import pytest
+import subprocess
+
+from lib.commands import local_cmd
+
+# Requirements:
+# - 1 XCP-ng host
+
+MYDIR = os.path.dirname(__file__)
+REF_DIR = os.path.join(MYDIR, "data")
+FSDIFF = os.path.realpath(f"{MYDIR}/../../scripts/xcpng-fs-diff.py")
+
+def _ref_name(host):
+    return f"{host.inventory['PRODUCT_VERSION']}-{host.firmware_type()}"
+
+# FIXME: not a test, rather a task done on a given (cached) host
+def test_fsdiff_mkref(hosts):
+    host, = hosts
+
+    logging.info("Extracting reference data from %s", host.hostname_or_ip)
+    process = local_cmd([FSDIFF,
+                         "--reference-host", host.hostname_or_ip,
+                         "--save-reference", os.path.join(REF_DIR, _ref_name(host)),
+                         ])
+
+def test_fsdiff_against_ref(hosts):
+    host, = hosts
+
+    ref_file = os.path.join(REF_DIR, _ref_name(host))
+    logging.info("Comparing %s with reference data", host.hostname_or_ip)
+    process = local_cmd([FSDIFF,
+                         "--load-reference", ref_file,
+                         "--test-host", host.hostname_or_ip,
+                         ])