Fix for Syself tests by cluster admin bound to system:masters group to run tests

Tests/README.md

@@ -20,6 +20,8 @@ At the moment, there are two cloud layers that can be checked: IaaS and KaaS.
 For both types of checks, the exit code indicates success (0) or failure (!= 0).
 You can also request a YAML report using the option `-o OUTPUT.yaml`
 
+### Environment configuration (`env.yaml`)
+
 Some tests can be adjusted to the environment by using variables. These variables
 can be specified in `$HOME/.config/scs/env.yaml`. Use the subject as a key for a
 dictionary for the variables. For instance, with a subject `openstack` and two
@@ -31,6 +33,16 @@ openstack:
     external_gateway_net_id: abcdef12-3456-7890-abcd-ef1234567890
 ```
 
+Similarly, for KaaS environments requiring specific test execution permissions
+(such as mounting an administrative kubeconfig into the Sonobuoy e2e test pod
+for subjects requiring `system:masters` group membership in Kubernetes 1.36),
+you can declare `sonobuoy_fix`:
+
+```yaml
+syself-1.36:
+    sonobuoy_fix: inject-kubeconfig
+```
+
 ### IaaS checks
 
 Install IaaS-specific requirements:
@@ -81,6 +93,12 @@ Additionally, the directory `sono-results` will be generated. It contains a JUni
 `plugins/e2e/results/global/junit_01.xml`. You can render it to HTML with a tool like junit2html.
 This might give you hints as to why a test failed.
 
+For environments requiring specific test execution workarounds (such as mounting
+the admin kubeconfig into the e2e test pod for subjects requiring `system:masters`
+group membership in Kubernetes 1.36), refer to the
+[Environment configuration (`env.yaml`)](#environment-configuration-envyaml)
+section above to configure `sonobuoy_fix` in `$HOME/.config/scs/env.yaml`.
+
 ## Usage information (help output)
 
 ```text

Tests/kaas/k8s_test.py

@@ -13,6 +13,7 @@
 import os.path
 import sys
 
+import yaml
 
 from scs_0210_version_policy.k8s_version_policy import version_policy_check
 from sonobuoy_handler.sonobuoy_handler import SonobuoyHandler
@@ -29,6 +30,20 @@
 SCS_SONOBUOY_CONFIG_PATH = os.path.join(BASEDIR, 'scs-sonobuoy-config-v1.yaml')
 
 logger = logging.getLogger(__name__)
+ENVPATH = os.path.join(os.path.expanduser('~'), '.config', 'scs', 'env.yaml')
+
+
+def _load_env_yaml_section(path, subject):
+    if not os.path.isfile(path):
+        return {}
+    with open(path, "rb") as fileobj:
+        env_yaml = yaml.load(fileobj, Loader=yaml.SafeLoader)
+    if not isinstance(env_yaml, dict):
+        raise RuntimeError("env.yaml config file available, but not of type dict!")
+    result = env_yaml.get(subject, {})
+    if not result:
+        logger.debug("env.yaml config file available, but no values declared.")
+    return result
 
 
 def usage(rcode=1, file=sys.stderr):
@@ -97,10 +112,12 @@ def compute_sono_args(self, *args):
 
 
 def run_sono(config, testcase, *args):
+    sonobuoy_fix = _load_env_yaml_section(path=ENVPATH, subject=config.subject).get('sonobuoy_fix', 'none')
     return_code = SonobuoyHandler(
         SCS_SONOBUOY_CONFIG_PATH, testcase, config.kubeconfig_path,
         result_dir_name=os.path.join(os.path.dirname(config.kubeconfig_path), testcase),
         args=config.compute_sono_args(*args),
+        sonobuoy_fix=sonobuoy_fix,
     ).run()
     return return_code == 0
 

Tests/kaas/sonobuoy_handler/run_sonobuoy.py

@@ -22,16 +22,24 @@ def cli():
 @click.option("-c", "--check", "check_name", type=str, default="sonobuoy_executor", help="this MUST be the same name as the id in 'scs-compatible-kaas.yaml'",)
 @click.option("--scs-sonobuoy-config", "scs_sonobuoy_config_yaml", required=True, type=click.Path(exists=True), help="Configuration for Sonobuoy (YAML format)")
 @click.option('--execution-mode', 'mode', type=click.Choice(('serial', 'parallel', 'dry')), default='serial')
+@click.option("--sonobuoy-fix", "fix", type=click.Choice(('none', 'inject-kubeconfig')), default='none', help="Sonobuoy fix strategy to apply")
 @click.option("-a", "--arg", "args", multiple=True)
-def sonobuoy_run(kubeconfig, result_dir_name, check_name, scs_sonobuoy_config_yaml, mode, args):
+def sonobuoy_run(kubeconfig, result_dir_name, check_name, scs_sonobuoy_config_yaml, mode, fix, args):
     if mode == 'parallel':
         # This is merely a shortcut to simplify commandline in scs-compatible-kaas.yaml
         # For more on parallel execution, see https://github.com/vmware-tanzu/sonobuoy/issues/1435
         args += ('--e2e-parallel=true', )
     elif mode == 'dry':
         # mutually exclusive with parallel, it turns out!
         args += ('--plugin-env e2e.E2E_DRYRUN=true', )
-    sonobuoy_handler = SonobuoyHandler(scs_sonobuoy_config_yaml, check_name, kubeconfig, result_dir_name, args)
+    sonobuoy_handler = SonobuoyHandler(
+        scs_sonobuoy_config_yaml,
+        check_name,
+        kubeconfig,
+        result_dir_name,
+        args,
+        sonobuoy_fix=fix,
+    )
     sys.exit(sonobuoy_handler.run())
 
 

Tests/kaas/sonobuoy_handler/sonobuoy_handler.py

@@ -1,3 +1,4 @@
+import base64
 from collections import Counter
 import logging
 import re
@@ -30,6 +31,142 @@ def _fmt_result(counter):
     return ', '.join(f"{counter.get(key, 0)} {key}" for key in ('passed', 'failed', 'failed_ok', 'skipped'))
 
 
+def _get_plugins_cm(docs):
+    """Locate the sonobuoy-plugins-cm ConfigMap document in the generated sonobuoy manifest."""
+    for doc in docs:
+        if not doc:
+            continue
+        if doc.get("kind") != "ConfigMap":
+            continue
+        if doc.get("metadata", {}).get("name") != "sonobuoy-plugins-cm":
+            continue
+        return doc
+    raise RuntimeError("Failed to locate 'sonobuoy-plugins-cm' ConfigMap in sonobuoy manifest")
+
+
+def _patch_e2e_plugin_cm(plugins_cm):
+    """Patch the 'e2e' plugin definition in sonobuoy-plugins-cm to mount and use admin kubeconfig."""
+    data_map = plugins_cm.get("data", {})
+    for plugin_key, plugin_raw in data_map.items():
+        plugin_data = yaml.safe_load(plugin_raw)
+        if not isinstance(plugin_data, dict):
+            continue
+        if plugin_data.get("sonobuoy-config", {}).get("plugin-name") != "e2e":
+            continue
+
+        # Mount the admin-kubeconfig secret into the e2e test pod
+        pod_spec = plugin_data.setdefault("podSpec", {})
+        pod_spec["volumes"] = pod_spec.get("volumes", []) + [
+            {"name": "admin-kubeconfig-vol", "secret": {"secretName": "admin-kubeconfig"}}
+        ]
+        spec = plugin_data.setdefault("spec", {})
+        spec["volumeMounts"] = spec.get("volumeMounts", []) + [
+            {"name": "admin-kubeconfig-vol", "mountPath": "/etc/kubernetes/admin.conf", "subPath": "admin.conf"}
+        ]
+
+        # Configure e2e test to use the admin kubeconfig:
+        # 1. Pass --kubeconfig via E2E_EXTRA_ARGS (later flag wins in Go test runner)
+        # 2. Set KUBECONFIG env var for subprocesses and client-go fallback
+        envs = spec.setdefault("env", [])
+        extra_kubeconfig_arg = "--kubeconfig=/etc/kubernetes/admin.conf"
+        for env in envs:
+            if env.get("name") == "E2E_EXTRA_ARGS":
+                env["value"] = f"{env.get('value', '').strip()} {extra_kubeconfig_arg}".strip()
+                break
+        else:
+            envs.append({"name": "E2E_EXTRA_ARGS", "value": extra_kubeconfig_arg})
+
+        for env in envs:
+            if env.get("name") == "KUBECONFIG":
+                env["value"] = "/etc/kubernetes/admin.conf"
+                break
+        else:
+            envs.append({"name": "KUBECONFIG", "value": "/etc/kubernetes/admin.conf"})
+
+        data_map[plugin_key] = yaml.dump(plugin_data)
+        return
+
+    raise RuntimeError("Failed to locate and patch 'e2e' plugin inside sonobuoy-plugins-cm ConfigMap")
+
+
+class SonoFixBase:
+    """Base strategy for executing Sonobuoy test runs."""
+
+    def __init__(self, handler):
+        self.handler = handler
+
+    def run(self):
+        raise NotImplementedError
+
+
+class SonoFixNone(SonoFixBase):
+    """Default strategy: run Sonobuoy directly without manifest modifications."""
+
+    def run(self):
+        self.handler._invoke_sonobuoy("run", "--wait", *self.handler.args)
+
+
+class SonoFixInjectKubeconfig(SonoFixBase):
+    """Strategy to inject admin kubeconfig into Sonobuoy e2e test pod."""
+
+    def run(self):
+        logger.info("Generating Sonobuoy manifest with mounted admin kubeconfig")
+        try:
+            gen_proc = self.handler._invoke_sonobuoy(
+                "gen", *self.handler.args, stdout=subprocess.PIPE, text=True
+            )
+        except subprocess.CalledProcessError as exc:
+            logger.error(f"sonobuoy gen failed: {exc}")
+            raise
+
+        docs = [d for d in yaml.safe_load_all(gen_proc.stdout) if d is not None]
+
+        # Extract namespace from manifest and set it on handler so subsequent
+        # run, wait, retrieve, and delete calls target this namespace
+        namespace_idx = next(
+            (idx for idx, d in enumerate(docs) if d.get("kind") == "Namespace"), None
+        )
+        if namespace_idx is None:
+            raise RuntimeError("generated sonobuoy manifest contains no Namespace")
+        self.handler.namespace = docs[namespace_idx]["metadata"]["name"]
+        logger.debug(f"sonobuoy namespace: {self.handler.namespace}")
+
+        # Insert Secret containing admin kubeconfig into manifest so sonobuoy lifecycle owns it
+        with open(self.handler.kubeconfig_path, "rb") as kf:
+            kdata = base64.b64encode(kf.read()).decode("ascii")
+
+        secret_doc = {
+            "apiVersion": "v1",
+            "kind": "Secret",
+            "metadata": {
+                "name": "admin-kubeconfig",
+                "namespace": self.handler.namespace,
+                "labels": {
+                    "component": "sonobuoy",
+                },
+            },
+            "data": {
+                "admin.conf": kdata,
+            },
+        }
+        docs.insert(namespace_idx + 1, secret_doc)
+
+        # Patch e2e plugin in sonobuoy-plugins-cm ConfigMap
+        plugins_cm = _get_plugins_cm(docs)
+        _patch_e2e_plugin_cm(plugins_cm)
+
+        modified_manifest = yaml.dump_all(docs)
+
+        # Launch Sonobuoy via stdin (-f -)
+        self.handler._invoke_sonobuoy("run", "-f", "-", input=modified_manifest, text=True)
+
+        # Waiting is unconditional, but it has to be a separate step: `run -f - --wait`
+        # makes sonobuoy re-read the manifest from stdin to determine the namespace,
+        # which is already consumed by then. Bare `--wait` means 1440 minutes; without
+        # it the flag defaults to 0, which means "do not wait" and exits non-zero.
+        self.handler._invoke_sonobuoy("wait", "--wait")
+
+
 class SonobuoyHandler:
     """
     A class that handles both the execution of sonobuoy and
@@ -46,6 +183,7 @@ def __init__(
         kubeconfig=None,
         result_dir_name=None,
         args=(),
+        sonobuoy_fix="none",
     ):
         self.check_name = check_name
         logger.debug(f"kubeconfig: {kubeconfig} ")
@@ -63,16 +201,33 @@ def __init__(
         if not os.path.exists(scs_sonobuoy_config_yaml):
             raise RuntimeError(f"scs_sonobuoy_config_yaml {scs_sonobuoy_config_yaml} does not exist.")
         self.scs_sonobuoy_config_yaml = scs_sonobuoy_config_yaml
-        self.args = (arg0 for arg in args for arg0 in shlex.split(str(arg)))
+        self.args = tuple(arg0 for arg in args for arg0 in shlex.split(str(arg)))
+        # Dynamically populated by fix strategies (e.g., SonoFixInjectKubeconfig)
+        # to ensure subsequent wait/retrieve/delete calls target the exact namespace.
+        self.namespace = None
+
+        if sonobuoy_fix == "inject-kubeconfig":
+            self.sonobuoy_fix = SonoFixInjectKubeconfig(self)
+        elif sonobuoy_fix in (None, "none", ""):
+            self.sonobuoy_fix = SonoFixNone(self)
+        else:
+            raise ValueError(f"Unknown sonobuoy fix: {sonobuoy_fix}")
+
+    @property
+    def sono_args(self):
+        cmd = (self.sonobuoy, "--kubeconfig", self.kubeconfig_path)
+        if self.namespace:
+            cmd += ("-n", self.namespace)
+        return cmd
 
-    def _invoke_sonobuoy(self, *args, **kwargs):
-        inv_args = (self.sonobuoy, "--kubeconfig", self.kubeconfig_path) + args
+    def _invoke_sonobuoy(self, *args, stdout=sys.stderr, **kwargs):
+        inv_args = self.sono_args + args
         logger.debug(f'invoking {" ".join(inv_args)}')
         # we had capture_output=True, but I don't see why -- let the caller decide
-        return subprocess.run(args=inv_args, check=True, stdout=sys.stderr, **kwargs)
+        return subprocess.run(args=inv_args, check=True, stdout=stdout, **kwargs)
 
     def _sonobuoy_run(self):
-        self._invoke_sonobuoy("run", "--wait", *self.args)
+        self.sonobuoy_fix.run()
 
     def _sonobuoy_delete(self):
         self._invoke_sonobuoy("delete", "--wait")

playbooks/.config/scs/env.yaml

@@ -0,0 +1,2 @@
+syself-1.36:
+  sonobuoy_fix: inject-kubeconfig