From 47821bf138b5dc5bea0ff9c351c854acb63e9c75 Mon Sep 17 00:00:00 2001
From: Yann Dirson <yann.dirson@vates.tech>
Date: Wed, 11 Sep 2024 22:32:13 +0200
Subject: [PATCH] WIP install: new test to check against reference files

Tests for generation of the reference files, and for checking against them.

FIXME: needs reference files for releases
---
 lib/host.py                      |  5 +++++
 scripts/runtests-get-fsdiff-ref  | 31 +++++++++++++++++++++++++++
 scripts/runtests-postinstall.sh  |  6 ++++++
 scripts/runtests-postupgrade.sh  |  7 +++++++
 tests/fs-diff/test_fsdiff_ref.py | 36 ++++++++++++++++++++++++++++++++
 5 files changed, 85 insertions(+)
 create mode 100644 scripts/runtests-get-fsdiff-ref
 create mode 100644 tests/fs-diff/test_fsdiff_ref.py

diff --git a/lib/host.py b/lib/host.py
index 6a434d38..4b2c5950 100644
--- a/lib/host.py
+++ b/lib/host.py
@@ -600,3 +600,8 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def firmware_type(self):
+        retcode = self.ssh(['test', '-d', '/sys/firmware/efi/'],
+                           check=False, simple_output=False).returncode
+        return "uefi" if retcode == 0 else "bios"
diff --git a/scripts/runtests-get-fsdiff-ref b/scripts/runtests-get-fsdiff-ref
new file mode 100644
index 00000000..772da922
--- /dev/null
+++ b/scripts/runtests-get-fsdiff-ref
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+# extract reference dumps from install
+
+# needs at least --hosts=$NEST
+# needs runtest-install output cache
+
+VERSION="83nightly"
+
+. $(dirname $0)/lib.sh
+
+GITREV=$(git rev-parse HEAD)
+
+declare -A FSDIFFCONFS=()
+for conf in "${TESTCONFS[@]}"; do
+    IFS=- read fw sr < <(echo "$conf")
+    [ "$media" = "iso" -a "$sr" = "ext" ] || continue
+    FSDIFFCONFS[$fw-$VERSION]=$conf
+done
+
+for fsdiffconf in "${!FSDIFFCONFS[@]}"; do # "in FSDIFFCONFS.keys"
+    IFS=- read fw version < <(echo "$conf")
+    # get ref data for install
+    run_pytest \
+        --hosts="cache://install.test::Nested::boot_inst[$fw-$version-host1-iso-ext]-vm1-$GITREV" \
+        --log-file=test-fsdiffref-$conf.log \
+        tests/fs-diff/test_fsdiff_ref.py::test_fsdiff_mkref
+done
+
+report_failures
diff --git a/scripts/runtests-postinstall.sh b/scripts/runtests-postinstall.sh
index ce4a3c3f..a33110a5 100755
--- a/scripts/runtests-postinstall.sh
+++ b/scripts/runtests-postinstall.sh
@@ -18,6 +18,12 @@ for conf in "${TESTCONFS[@]}"; do
         --runner "./jobs.py run postinstall cache://${host1},cache://${host2}" \
         --log-file=test-postinstall-$conf.log \
         "$@"
+
+    run_pytest \
+        --hosts="cache://install.test::Nested::boot_inst[$fw-83nightly-host1-iso-ext]-vm1-$GITREV" \
+        --log-file=test-fsdiff-$conf.log \
+        "$@" \
+        tests/fs-diff/test_fsdiff_ref.py::test_fsdiff_against_ref
 done
 
 report_failures
diff --git a/scripts/runtests-postupgrade.sh b/scripts/runtests-postupgrade.sh
index c35e11c7..5952fd37 100755
--- a/scripts/runtests-postupgrade.sh
+++ b/scripts/runtests-postupgrade.sh
@@ -37,6 +37,13 @@ for conf in "${REFCONFS[@]}"; do
             # FIXME postinstall-intrapool-migrate (with TLS) needs a 2-host pool
             ;;
     esac
+
+    # FIXME: check fsdiff with TLS activated, since that's what the 8.3+ ref have
+
+    run_pytest \
+        --hosts="cache://${host1}" \
+        --log-file=test-fsdiff-$conf.log \
+        tests/fs-diff/test_fsdiff_ref.py::test_fsdiff_against_ref
 done
 
 report_failures
diff --git a/tests/fs-diff/test_fsdiff_ref.py b/tests/fs-diff/test_fsdiff_ref.py
new file mode 100644
index 00000000..2911e009
--- /dev/null
+++ b/tests/fs-diff/test_fsdiff_ref.py
@@ -0,0 +1,36 @@
+import logging
+import os
+import pytest
+import subprocess
+
+from lib.commands import local_cmd
+
+# Requirements:
+# - 1 XCP-ng host
+
+MYDIR = os.path.dirname(__file__)
+REF_DIR = os.path.join(MYDIR, "data")
+FSDIFF = os.path.realpath(f"{MYDIR}/../../scripts/xcpng-fs-diff.py")
+
+def _ref_name(host):
+    return f"{host.inventory['PRODUCT_VERSION']}-{host.firmware_type()}"
+
+# FIXME: not a test, rather a task done on a given (cached) host
+def test_fsdiff_mkref(hosts):
+    host, = hosts
+
+    logging.info("Extracting reference data from %s", host.hostname_or_ip)
+    process = local_cmd([FSDIFF,
+                         "--reference-host", host.hostname_or_ip,
+                         "--save-reference", os.path.join(REF_DIR, _ref_name(host)),
+                         ])
+
+def test_fsdiff_against_ref(hosts):
+    host, = hosts
+
+    ref_file = os.path.join(REF_DIR, _ref_name(host))
+    logging.info("Comparing %s with reference data", host.hostname_or_ip)
+    process = local_cmd([FSDIFF,
+                         "--load-reference", ref_file,
+                         "--test-host", host.hostname_or_ip,
+                         ])