[SYCL][E2E] Implement support for REQUIRES in build-only mode

sycl/test-e2e/format.py

@@ -52,6 +52,29 @@ def parse_min_intel_driver_req(line_number, line, output):
     return output
 
 
+def parse_bool_literal(line_number, line, output):
+    """
+    Input looks like this:
+    # KEYWORD: <"true" | "false">
+    """
+    if output is None:
+        output = []
+
+    line = line.strip()
+    if line == "true":
+        output.append(True)
+    elif line == "false":
+        output.append(False)
+    else:
+        raise ValueError(
+            "Unexpected value '{}', expected boolean literal 'true' or 'false'".format(
+                line
+            )
+        )
+
+    return output
+
+
 class SYCLEndToEndTest(lit.formats.ShTest):
     def parseTestScript(self, test):
         """This is based on lit.TestRunner.parseIntegratedTestScript but we
@@ -67,24 +90,92 @@ def parseTestScript(self, test):
                         "REQUIRES-INTEL-DRIVER:",
                         ParserKind.CUSTOM,
                         parse_min_intel_driver_req,
-                    )
+                    ),
+                    IntegratedTestKeywordParser(
+                        "ENABLE_RUN_REQUIRES:",
+                        ParserKind.CUSTOM,
+                        parse_bool_literal,
+                    ),
+                    IntegratedTestKeywordParser("RUN_XFAIL:", ParserKind.BOOLEAN_EXPR),
+                    IntegratedTestKeywordParser(
+                        "RUN_REQUIRES:", ParserKind.BOOLEAN_EXPR
+                    ),
+                    IntegratedTestKeywordParser(
+                        "RUN_UNSUPPORTED:", ParserKind.BOOLEAN_EXPR
+                    ),
                 ],
                 require_script=True,
             )
         except ValueError as e:
-            return lit.Test.Result(Test.UNRESOLVED, str(e))
+            return lit.Test.Result(lit.Test.UNRESOLVED, str(e))
         script = parsed["RUN:"] or []
         assert parsed["DEFINE:"] == script
         assert parsed["REDEFINE:"] == script
 
-        test.xfails += parsed["XFAIL:"] or []
-        test.requires += test.config.required_features
-        test.requires += parsed["REQUIRES:"] or []
-        test.unsupported += test.config.unsupported_features
-        test.unsupported += parsed["UNSUPPORTED:"] or []
+        if parsed["ENABLE_RUN_REQUIRES:"] is None:
+            enable_run_requires = test.config.enable_run_requires
+        elif len(parsed["ENABLE_RUN_REQUIRES:"]) > 1:
+            return lit.Test.Result(
+                lit.Test.UNRESOLVED, "Test has more than one ENABLE_RUN_REQUIRES lines"
+            )
+        else:
+            enable_run_requires = parsed["ENABLE_RUN_REQUIRES:"][0]
+
+        if not enable_run_requires:
+            for kw in ["RUN_XFAIL", "RUN_REQUIRES", "RUN_UNSUPPORTED"]:
+                if parsed[kw + ":"] is not None:
+                    return lit.Test.Result(
+                        lit.Test.UNRESOLVED,
+                        "Directive %s requires ENABLE_RUN_REQUIRES: true",
+                    )
+
+        if test.config.enable_run_requires:
+            test.requires += test.config.required_features
+            test.unsupported += test.config.unsupported_features
+
+            test.run_requires = test.config.run_required_features
+            test.run_unsupported = test.config.run_unsupported_features
+        else:
+            test.run_requires = test.config.required_features
+            test.run_unsupported = test.config.unsupported_features
+
+        if enable_run_requires:
+            test.xfails += parsed["XFAIL:"] or []
+            test.requires += parsed["REQUIRES:"] or []
+            test.unsupported += parsed["UNSUPPORTED:"] or []
+
+            test.run_xfails = parsed["RUN_XFAIL:"] or []
+            test.run_requires += parsed["RUN_REQUIRES:"] or []
+            test.run_unsupported += parsed["RUN_UNSUPPORTED:"] or []
+        else:
+            test.run_xfails = parsed["XFAIL:"] or []
+            test.run_requires = parsed["REQUIRES:"] or []
+            test.run_unsupported = parsed["UNSUPPORTED:"] or []
 
         test.intel_driver_req = parsed["REQUIRES-INTEL-DRIVER:"]
 
+        if test.config.enable_run_requires or enable_run_requires:
+            # Enforce REQUIRES:
+            missing_required_features = test.getMissingRequiredFeatures()
+            if missing_required_features:
+                msg = ", ".join(missing_required_features)
+                return lit.Test.Result(
+                    lit.Test.UNSUPPORTED,
+                    "Test requires the following unavailable " "features: %s" % msg,
+                )
+            # Enforce UNSUPPORTED:
+            unsupported_features = test.getUnsupportedFeatures()
+            if unsupported_features:
+                msg = ", ".join(unsupported_features)
+                return lit.Test.Result(
+                    lit.Test.UNSUPPORTED,
+                    "Test does not support the following features "
+                    "and/or targets: %s" % msg,
+                )
+        else:
+            assert len(test.requires) == 0
+            assert len(test.unsupported) == 0
+
         return script
 
     def getMatchedFromList(self, features, alist):
@@ -102,7 +193,7 @@ def select_devices_for_test(self, test):
             if test.getMissingRequiredFeaturesFromList(features):
                 continue
 
-            if self.getMatchedFromList(features, test.unsupported):
+            if self.getMatchedFromList(features, test.run_unsupported):
                 continue
 
             driver_ok = True
@@ -128,14 +219,14 @@ def select_devices_for_test(self, test):
         #
         # TODO: What if the entire list of devices consists of XFAILs only?
 
-        if "*" in test.xfails:
+        if "*" in test.run_xfails:
             return []
 
         devices_without_xfail = [
             d
             for d in devices
             if not self.getMatchedFromList(
-                test.config.sycl_dev_features[d], test.xfails
+                test.config.sycl_dev_features[d], test.run_xfails
             )
         ]
 
@@ -154,7 +245,10 @@ def execute(self, test, litConfig):
         devices_for_test = []
         triples = set()
         if test.config.test_mode == "build-only":
-            if "build-and-run-mode" in test.requires or "true" in test.unsupported:
+            if (
+                "build-and-run-mode" in test.run_requires
+                or "true" in test.run_unsupported
+            ):
                 return lit.Test.Result(
                     lit.Test.UNSUPPORTED, "Test unsupported for this environment"
                 )
@@ -253,7 +347,7 @@ def get_extra_env(sycl_devices):
             )
 
             ignore_line_filtering = (
-                "build-and-run-mode" in test.requires
+                "build-and-run-mode" in test.run_requires
                 and test.config.fallback_build_run_only
             )
             if not ignore_line_filtering and (
@@ -328,8 +422,8 @@ def get_extra_env(sycl_devices):
 
         # Single device - might be an XFAIL.
         device = devices_for_test[0]
-        if "*" in test.xfails or self.getMatchedFromList(
-            test.config.sycl_dev_features[device], test.xfails
+        if "*" in test.run_xfails or self.getMatchedFromList(
+            test.config.sycl_dev_features[device], test.run_xfails
         ):
             if result.code is lit.Test.PASS:
                 result.code = lit.Test.XPASS

sycl/test-e2e/lit.cfg.py

@@ -37,6 +37,18 @@
 config.required_features = []
 config.unsupported_features = []
 
+# Enable RUN_REQUIRES, RUN_UNSUPPORTED and RUN_XFAIL,
+# and change the meaning of REQUIRES, UNSUPPORTED and XFAIL to apply to both build and run.
+# Per-device features are only available in RUN_* directives in this mode.
+# Can be overridden by lit.local.cfg files. If set in lit.local.cfg then the restriction above
+# will also apply to config.required_features and config.unsupported_features.
+# Can be overridden per-test with `ENABLE_RUN_REQUIRES: <true | false>`
+config.enable_run_requires = False
+
+# To be filled by lit.local.cfg files. Ignored if enable_run_requires = False
+config.run_required_features = []
+config.run_unsupported_features = []
+
 # test-mode: Set if tests should run normally or only build/run
 config.test_mode = lit_config.params.get("test-mode", "full")
 config.fallback_build_run_only = False