diff --git a/sycl/test-e2e/format.py b/sycl/test-e2e/format.py index eac6ebe0ba2ed..729a9647309a1 100644 --- a/sycl/test-e2e/format.py +++ b/sycl/test-e2e/format.py @@ -52,6 +52,29 @@ def parse_min_intel_driver_req(line_number, line, output): return output +def parse_bool_literal(line_number, line, output): + """ + Input looks like this: + # KEYWORD: <"true" | "false"> + """ + if output is None: + output = [] + + line = line.strip() + if line == "true": + output.append(True) + elif line == "false": + output.append(False) + else: + raise ValueError( + "Unexpected value '{}', expected boolean literal 'true' or 'false'".format( + line + ) + ) + + return output + + class SYCLEndToEndTest(lit.formats.ShTest): def parseTestScript(self, test): """This is based on lit.TestRunner.parseIntegratedTestScript but we @@ -67,24 +90,92 @@ def parseTestScript(self, test): "REQUIRES-INTEL-DRIVER:", ParserKind.CUSTOM, parse_min_intel_driver_req, - ) + ), + IntegratedTestKeywordParser( + "ENABLE_RUN_REQUIRES:", + ParserKind.CUSTOM, + parse_bool_literal, + ), + IntegratedTestKeywordParser("RUN_XFAIL:", ParserKind.BOOLEAN_EXPR), + IntegratedTestKeywordParser( + "RUN_REQUIRES:", ParserKind.BOOLEAN_EXPR + ), + IntegratedTestKeywordParser( + "RUN_UNSUPPORTED:", ParserKind.BOOLEAN_EXPR + ), ], require_script=True, ) except ValueError as e: - return lit.Test.Result(Test.UNRESOLVED, str(e)) + return lit.Test.Result(lit.Test.UNRESOLVED, str(e)) script = parsed["RUN:"] or [] assert parsed["DEFINE:"] == script assert parsed["REDEFINE:"] == script - test.xfails += parsed["XFAIL:"] or [] - test.requires += test.config.required_features - test.requires += parsed["REQUIRES:"] or [] - test.unsupported += test.config.unsupported_features - test.unsupported += parsed["UNSUPPORTED:"] or [] + if parsed["ENABLE_RUN_REQUIRES:"] is None: + enable_run_requires = test.config.enable_run_requires + elif len(parsed["ENABLE_RUN_REQUIRES:"]) > 1: + return lit.Test.Result( + lit.Test.UNRESOLVED, "Test has more than one ENABLE_RUN_REQUIRES lines" + ) + else: + enable_run_requires = parsed["ENABLE_RUN_REQUIRES:"][0] + + if not enable_run_requires: + for kw in ["RUN_XFAIL", "RUN_REQUIRES", "RUN_UNSUPPORTED"]: + if parsed[kw + ":"] is not None: + return lit.Test.Result( + lit.Test.UNRESOLVED, + "Directive %s requires ENABLE_RUN_REQUIRES: true", + ) + + if test.config.enable_run_requires: + test.requires += test.config.required_features + test.unsupported += test.config.unsupported_features + + test.run_requires = test.config.run_required_features + test.run_unsupported = test.config.run_unsupported_features + else: + test.run_requires = test.config.required_features + test.run_unsupported = test.config.unsupported_features + + if enable_run_requires: + test.xfails += parsed["XFAIL:"] or [] + test.requires += parsed["REQUIRES:"] or [] + test.unsupported += parsed["UNSUPPORTED:"] or [] + + test.run_xfails = parsed["RUN_XFAIL:"] or [] + test.run_requires += parsed["RUN_REQUIRES:"] or [] + test.run_unsupported += parsed["RUN_UNSUPPORTED:"] or [] + else: + test.run_xfails = parsed["XFAIL:"] or [] + test.run_requires = parsed["REQUIRES:"] or [] + test.run_unsupported = parsed["UNSUPPORTED:"] or [] test.intel_driver_req = parsed["REQUIRES-INTEL-DRIVER:"] + if test.config.enable_run_requires or enable_run_requires: + # Enforce REQUIRES: + missing_required_features = test.getMissingRequiredFeatures() + if missing_required_features: + msg = ", ".join(missing_required_features) + return lit.Test.Result( + lit.Test.UNSUPPORTED, + "Test requires the following unavailable " "features: %s" % msg, + ) + # Enforce UNSUPPORTED: + unsupported_features = test.getUnsupportedFeatures() + if unsupported_features: + msg = ", ".join(unsupported_features) + return lit.Test.Result( + lit.Test.UNSUPPORTED, + "Test does not support the following features " + "and/or targets: %s" % msg, + ) + else: + assert len(test.requires) == 0 + assert len(test.unsupported) == 0 + return script def getMatchedFromList(self, features, alist): @@ -102,7 +193,7 @@ def select_devices_for_test(self, test): if test.getMissingRequiredFeaturesFromList(features): continue - if self.getMatchedFromList(features, test.unsupported): + if self.getMatchedFromList(features, test.run_unsupported): continue driver_ok = True @@ -128,14 +219,14 @@ def select_devices_for_test(self, test): # # TODO: What if the entire list of devices consists of XFAILs only? - if "*" in test.xfails: + if "*" in test.run_xfails: return [] devices_without_xfail = [ d for d in devices if not self.getMatchedFromList( - test.config.sycl_dev_features[d], test.xfails + test.config.sycl_dev_features[d], test.run_xfails ) ] @@ -154,7 +245,10 @@ def execute(self, test, litConfig): devices_for_test = [] triples = set() if test.config.test_mode == "build-only": - if "build-and-run-mode" in test.requires or "true" in test.unsupported: + if ( + "build-and-run-mode" in test.run_requires + or "true" in test.run_unsupported + ): return lit.Test.Result( lit.Test.UNSUPPORTED, "Test unsupported for this environment" ) @@ -253,7 +347,7 @@ def get_extra_env(sycl_devices): ) ignore_line_filtering = ( - "build-and-run-mode" in test.requires + "build-and-run-mode" in test.run_requires and test.config.fallback_build_run_only ) if not ignore_line_filtering and ( @@ -328,8 +422,8 @@ def get_extra_env(sycl_devices): # Single device - might be an XFAIL. device = devices_for_test[0] - if "*" in test.xfails or self.getMatchedFromList( - test.config.sycl_dev_features[device], test.xfails + if "*" in test.run_xfails or self.getMatchedFromList( + test.config.sycl_dev_features[device], test.run_xfails ): if result.code is lit.Test.PASS: result.code = lit.Test.XPASS diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index 5b60d93387b7a..452ccf8088dd9 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -37,6 +37,18 @@ config.required_features = [] config.unsupported_features = [] +# Enable RUN_REQUIRES, RUN_UNSUPPORTED and RUN_XFAIL, +# and change the meaning of REQUIRES, UNSUPPORTED and XFAIL to apply to both build and run. +# Per-device features are only available in RUN_* directives in this mode. +# Can be overridden by lit.local.cfg files. If set in lit.local.cfg then the restriction above +# will also apply to config.required_features and config.unsupported_features. +# Can be overridden per-test with `ENABLE_RUN_REQUIRES: ` +config.enable_run_requires = False + +# To be filled by lit.local.cfg files. Ignored if enable_run_requires = False +config.run_required_features = [] +config.run_unsupported_features = [] + # test-mode: Set if tests should run normally or only build/run config.test_mode = lit_config.params.get("test-mode", "full") config.fallback_build_run_only = False