[SYCL][E2E] Add test to check REQUIRES

llvm/utils/lit/lit/TestingConfig.py

@@ -2,6 +2,27 @@
 import sys
 
 
+# config.available_features is a list. This class behaves like a normal list,
+# but its "add()" method also checks if the feature being added is known, i.e.
+# contained in the set of all alowed features.
+class AvailableFeatures(set):
+    def __init__(self, *args, **kwargs):
+        from sycl_lit_allowed_features import get_sycl_lit_allowed_features
+        allowed_features = get_sycl_lit_allowed_features()
+        super().__init__(*args, **kwargs)
+        self.allowed_features = allowed_features
+
+    def add(self, entry):
+        if entry in self.allowed_features:
+            super().add(entry)
+        else:
+            raise ValueError(
+                "feature "
+                + entry
+                + " is not allowed. Please update the set in sycl/test-e2e/sycl_lit_allowed_features.py."
+            )
+
+
 class TestingConfig(object):
     """
     TestingConfig - Information on the tests inside a suite.

sycl/test-e2e/lit.cfg.py

@@ -14,6 +14,7 @@
 from lit.llvm import llvm_config
 from lit.llvm.subst import ToolSubst, FindTool
 
+
 # Configuration file for the 'lit' test runner.
 config.backend_to_target = {
     "level_zero": "target-spir",
@@ -33,6 +34,8 @@
     k: config.target_to_triple.get(v) for k, v in config.backend_to_target.items()
 }
 
+config.available_features = AvailableFeatures()
+
 # name: The name of this test suite.
 config.name = "SYCL"
 
@@ -227,6 +230,7 @@ def get_device_family_from_arch(arch):
             return device_family
     return None
 
+
 def check_igc_tag_and_add_feature():
     if os.path.isfile(config.igc_tag_file):
         with open(config.igc_tag_file, "r") as tag_file:
@@ -254,6 +258,7 @@ def check_igc_tag_and_add_feature():
 if lit_config.params.get("spirv-backend", False):
     config.available_features.add("spirv-backend")
 
+
 # Use this to make sure that any dynamic checks below are done in the build
 # directory and not where the sources are located. This is important for the
 # in-tree configuration (as opposite to the standalone one).
@@ -781,7 +786,7 @@ def open_check_file(file_name):
             gpu_intel_pvc_2T_device_id = "3029"
             _, device_id = line.strip().split(":", 1)
             device_id = device_id.strip()
-            if device_id == gpu_intel_pvc_1T_device_id:             
+            if device_id == gpu_intel_pvc_1T_device_id:
                 config.available_features.add("gpu-intel-pvc-1T")
             if device_id == gpu_intel_pvc_2T_device_id:
                 config.available_features.add("gpu-intel-pvc-2T")

sycl/test-e2e/sycl_lit_allowed_features.py

@@ -0,0 +1,134 @@
+# This script provides the complete set of all possible features that can be
+# used in XFAIL, UNSUPPORTED and REQUIRES.
+
+# To use:
+# from sycl_lit_allowed_features import get_sycl_lit_allowed_features
+# allowed_features = get_sycl_lit_allowed_features()
+
+# Note:
+# The set below (partial_set_of_features) is maintained manually. If the new
+# feature is NOT an aspect or an architecture - it should be added to this set.
+# And vice versa - if the feature was deleted, it also should be deleted from
+# this set, otherwise the feature is still treated as valid.
+#
+# Aspects and device architectures are added automatically and require no
+# additional changes.
+
+import os
+
+partial_set_of_features = {
+    # for completely disabled tests
+    "true",
+    # host OS:
+    "windows",
+    "system-windows",
+    "linux",
+    "system-linux",
+    # target device:
+    "cpu",
+    "gpu",
+    "accelerator",
+    # target backend:
+    "cuda",
+    "hip",
+    "opencl",
+    "level_zero",
+    "level-zero",
+    "native_cpu",
+    # target:
+    "target-nvidia",
+    "target-native_cpu",
+    "target-amd",
+    "target-spir",
+    "spirv-backend",
+    # tools:
+    "sycl-ls",
+    "cm-compiler",
+    "ocloc",
+    "opencl-aot",
+    "llvm-spirv",
+    "llvm-link",
+    # dev-kits:
+    "level_zero_dev_kit",
+    "cuda_dev_kit",
+    # manually-set features (deprecated, no new tests should use these features)
+    "gpu-intel-gen11",
+    "gpu-intel-gen12",
+    "gpu-intel-dg1",
+    "gpu-intel-dg2",
+    "gpu-intel-pvc",
+    "gpu-intel-pvc-vg",
+    "gpu-intel-pvc-1T",
+    "gpu-intel-pvc-2T",
+    "gpu-amd-gfx90a",
+    # any-device-is-:
+    "any-device-is-cpu",
+    "any-device-is-gpu",
+    "any-device-is-accelerator",
+    "any-device-is-cuda",
+    "any-device-is-hip",
+    "any-device-is-opencl",
+    "any-device-is-level_zero",
+    "any-device-is-native_cpu",
+    # any-target-is-:
+    "any-target-is-spir",
+    "any-target-is-native_cpu",
+    "any-target-is-nvidia",
+    "any-target-is-amd",
+    "any-target-is-native_cpu",
+    # sg-sizes (should we allow any sg-X?):
+    "sg-8",
+    "sg-16",
+    "sg-32",
+    # e2e-modes:
+    "run-mode",
+    "build-and-run-mode",
+    # miscellaneous:
+    "cl_options",
+    "opencl_icd",
+    "dump_ir",
+    "xptifw",
+    "has_ndebug",
+    "zstd",
+    "preview-breaking-changes-supported",
+    "vulkan",
+    "O0",
+    "ze_debug",
+    "igc-dev",
+    "enable-perf-tests",
+}
+
+
+# To parse .def files to get aspects and architectures
+def parse_defines(path, macro, prefix):
+    features = set()
+    with open(path, "r") as file:
+        for line in file:
+            if line.startswith(macro):
+                feature = line.split("(")[1].split(",")[0].strip()
+                features.add(f"{prefix}-{feature}")
+        return features
+
+
+def get_sycl_lit_allowed_features():
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    aspects = parse_defines(
+        current_dir + "/../include/sycl/info/aspects.def", "__SYCL_ASPECT", "aspect"
+    )
+    aspects_deprecated = parse_defines(
+        current_dir + "/../include/sycl/info/aspects_deprecated.def",
+        "__SYCL_ASPECT",
+        "aspect",
+    )
+    architectures = parse_defines(
+        current_dir
+        + "/../include/sycl/ext/oneapi/experimental/device_architecture.def",
+        "__SYCL_ARCHITECTURE",
+        "arch",
+    )
+
+    # Combine all sets
+    all_features = (
+        partial_set_of_features | aspects | aspects_deprecated | architectures
+    )
+    return all_features

sycl/test/e2e_test_requirements/check-correctness-of-requirements.cpp

@@ -0,0 +1,16 @@
+// This test checks that all "REQUIRES", "XFAIL" and "UNSUPPORTED" strings
+// contain the right feature names.
+//
+// If this test fails:
+// 1. there is some typo/deleted feature requested in the new/modified test.
+// 2. ...or, there is some new feature. In this case please update the set of
+//    features in sycl/test-e2e/sycl_lit_allowed_features.py.
+//
+// Get a set of all features passed to "REQUIRES", "XFAIL" and "UNSUPPORTED":
+// RUN: llvm-lit --show-used-features %S/../../test-e2e > %t
+//
+// Process this set using a python script:
+// RUN: python3 %S/check-correctness-of-requirements.py %t
+//
+// Small negative test
+// RUN: echo non-existing-feature-request > %t | not python3 %S/check-correctness-of-requirements.py %t

sycl/test/e2e_test_requirements/check-correctness-of-requirements.py

@@ -0,0 +1,23 @@
+# See check-correctness-of-requirements.cpp
+
+import os
+import sys
+
+input_data = sys.argv[1]
+current_dir = os.path.dirname(os.path.abspath(__file__))
+
+# See sycl/test-e2e/sycl_lit_allowed_features.py.
+sys.path.append(current_dir + "/../../test-e2e")
+from sycl_lit_allowed_features import get_sycl_lit_allowed_features
+
+allowed_features = get_sycl_lit_allowed_features()
+
+exit_code = 0
+with open(input_data, "r") as file:
+    requirements = set(file.read().split())
+    for requirement in requirements:
+        if not requirement in allowed_features:
+            exit_code = 1
+            print("Unsupported requirement: " + requirement)
+
+sys.exit(exit_code)