From d5a04e57b0cee1afbfaab48cc6123ab4684964e7 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran S Date: Wed, 12 Jun 2024 14:00:45 +0530 Subject: [PATCH] overlays: Implement overlays option Signed-off-by: Senthil Kumaran S --- lava_test_plans/__main__.py | 27 +++++++++++++++++- lava_test_plans/include/fastboot.jinja2 | 14 +++++++++ lava_test_plans/nfs.jinja2 | 14 +++++++++ lava_test_plans/qemu.jinja2 | 14 +++++++++ lava_test_plans/utils.py | 38 +++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 1 deletion(-) diff --git a/lava_test_plans/__main__.py b/lava_test_plans/__main__.py index 8814619e..784d4d11 100644 --- a/lava_test_plans/__main__.py +++ b/lava_test_plans/__main__.py @@ -35,7 +35,12 @@ from ruamel.yaml.composer import ComposerError from lava_test_plans import __version__ -from lava_test_plans.utils import get_context, validate_variables +from lava_test_plans.utils import ( + compression, + get_context, + overlay_action, + validate_variables, +) FORMAT = "[%(funcName)16s() ] %(message)s" logging.basicConfig(level=logging.INFO, format=FORMAT) @@ -297,6 +302,19 @@ def main(): type=int, default=logging.INFO, ) + parser.add_argument( + "--overlay", + default=[], + metavar="URL/String", + type=str, + help=( + "Tarball with overlay and optionally PATH to extract the tarball," + " default PATH '/'. Overlay can be specified multiple times" + ), + action=overlay_action, + nargs="+", + dest="overlays", + ) args = parser.parse_args() logger.setLevel(args.verbose) @@ -342,6 +360,11 @@ def main(): logger.error("QA_REPORTS_TOKEN and LAVA_TOKEN are missing") return 1 + overlays = [] + if args.overlays: + for index, item in enumerate(args.overlays): + overlays.append((f"overlay-{index:02}", item[0], item[1])) + lava_jobs = [] template_dirs = [ @@ -359,8 +382,10 @@ def main(): else StrictUndefined ), ) + j2_env.globals["compression"] = compression context = get_context(script_dirname, args.variables, args.overwrite_variables) context.update({"device_type": args.device_type}) + context.update({"overlays": overlays}) test_list = [] if args.test_plan: for test_plan in args.test_plan: diff --git a/lava_test_plans/include/fastboot.jinja2 b/lava_test_plans/include/fastboot.jinja2 index 4e687fb5..d804f382 100644 --- a/lava_test_plans/include/fastboot.jinja2 +++ b/lava_test_plans/include/fastboot.jinja2 @@ -168,6 +168,20 @@ reboot_to_fastboot: {{ reboot_to_fastboot }} {% if apply_overlay == "overlays" %} overlays: {% endif %} +{% if overlays %} + overlays: +{% endif %} +{% for name, overlay, dst in overlays %} + {{ name }}: + url: "{{ overlay }}" + format: {{ compression(overlay)[0] }} +{% if compression(overlay)[1] is not none %} + compression: {{ compression(overlay)[1] }} + path: "{{ dst }}" +{% else %} + path: "{{ dst }}{{ overlay.split('/')[-1] }}" +{% endif %} +{% endfor %} {% if OVERLAY_URL is defined %} over: url: {{OVERLAY_URL}} diff --git a/lava_test_plans/nfs.jinja2 b/lava_test_plans/nfs.jinja2 index 9f053f6b..bcb4f1c6 100644 --- a/lava_test_plans/nfs.jinja2 +++ b/lava_test_plans/nfs.jinja2 @@ -96,6 +96,20 @@ {% if apply_overlay == "overlays" %} overlays: {% endif %} +{% if overlays %} + overlays: +{% endif %} +{% for name, overlay, dst in overlays %} + {{ name }}: + url: "{{ overlay }}" + format: {{ compression(overlay)[0] }} +{% if compression(overlay)[1] is not none %} + compression: {{ compression(overlay)[1] }} + path: "{{ dst }}" +{% else %} + path: "{{ dst }}{{ overlay.split('/')[-1] }}" +{% endif %} +{% endfor %} {% if OVERLAY_URL is defined %} over: url: {{OVERLAY_URL}} diff --git a/lava_test_plans/qemu.jinja2 b/lava_test_plans/qemu.jinja2 index 7fdfc36e..8c196188 100644 --- a/lava_test_plans/qemu.jinja2 +++ b/lava_test_plans/qemu.jinja2 @@ -100,6 +100,20 @@ {% block rootfs_extra_args %} {% endblock rootfs_extra_args %} {# Process the following block if any overlays provided #} +{% if overlays %} + overlays: +{% endif %} +{% for name, overlay, dst in overlays %} + {{ name }}: + url: "{{ overlay }}" + format: {{ compression(overlay)[0] }} +{% if compression(overlay)[1] is not none %} + compression: {{ compression(overlay)[1] }} + path: "{{ dst }}" +{% else %} + path: "{{ dst }}{{ overlay.split('/')[-1] }}" +{% endif %} +{% endfor %} {% if OVERLAY_MODULES_URL is defined or OVERLAY_KSELFTEST_URL is defined or OVERLAY_PERF_URL is defined or diff --git a/lava_test_plans/utils.py b/lava_test_plans/utils.py index c89e1688..1492a533 100644 --- a/lava_test_plans/utils.py +++ b/lava_test_plans/utils.py @@ -8,6 +8,7 @@ import os +import argparse import logging from configobj import ConfigObj, ConfigObjError from ruamel.yaml import YAML @@ -61,3 +62,40 @@ def validate_variables( logger.error(f"Mandatory variables missing: {var_diff}") return 1 return 0 + + +class overlay_action(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + entries = len(values) + + pairs = getattr(namespace, self.dest, []) + + if entries > 2: + parser.error( + f"More than 2 arguments passed for {self.dest} options. Please check help options" + ) + + if entries == 1: + pairs.append([values[0], "/"]) + else: + pairs.append([values[0], values[1]]) + setattr(namespace, self.dest, pairs) + + +COMPRESSIONS = { + ".tar.xz": ("tar", "xz"), + ".tar.gz": ("tar", "gz"), + ".tgz": ("tar", "gz"), + ".gz": (None, "gz"), + ".xz": (None, "xz"), + ".zst": (None, "zstd"), + ".py": ("file", None), + ".sh": ("file", None), +} + + +def compression(path): + for ext, ret in COMPRESSIONS.items(): + if path.endswith(ext): + return ret + return (None, None)