TESTING: Make the test failures fatal and resolve all of them

.github/workflows/compilers.yml

@@ -70,6 +70,7 @@ jobs:
       CC: ${{ matrix.toolchain.CC }}
       FFLAGS: ${{ matrix.toolchain.FFLAGS }}
       CFLAGS: ${{ matrix.toolchain.CFLAGS }}
+      JOB_ID: ${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }}-${{ matrix.lib_type }}
 
     strategy:
       fail-fast: false
@@ -380,6 +381,7 @@ jobs:
           -D BUILD_TESTING:BOOL=ON \
           -D LAPACKE_WITH_TMG:BOOL=ON \
           -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }} \
+          -D LAPACK_TESTING_JUNIT_JOB_ID="${JOB_ID}" \
           "${extra[@]}"
 
     - name: Build
@@ -409,7 +411,8 @@ jobs:
           -D LAPACKE:BOOL=ON ^
           -D BUILD_TESTING:BOOL=ON ^
           -D LAPACKE_WITH_TMG:BOOL=ON ^
-          -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }}
+          -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }} ^
+          -D LAPACK_TESTING_JUNIT_JOB_ID=%JOB_ID%
 
     - name: Build (Windows)
       if: ${{ runner.os == 'Windows' }}
@@ -453,7 +456,7 @@ jobs:
         files: build/lapack_testing_junit.xml
         # The report is named above, so there is nothing to search the tree for.
         disable_search: true
-        flags: tests-${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }}-${{ matrix.lib_type }}
+        flags: tests-${{ env.JOB_ID }}
         name: ${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }} (${{ matrix.lib_type }})
         fail_ci_if_error: true
         verbose: true

.github/workflows/special.yml

@@ -67,6 +67,7 @@ jobs:
       FFLAGS: "-Wall -Wno-unused-dummy-argument -Wno-unused-variable -Wno-unused-label -Werror=conversion -fimplicit-none -frecursive -fcheck=all -fopenmp"
       FC: ${{ matrix.toolchain.FC }}
       CC: ${{ matrix.toolchain.CC }}
+      JOB_ID: openmp-${{ matrix.toolchain.os }}-${{ matrix.lib_type }}
 
     strategy:
       fail-fast: false
@@ -102,6 +103,8 @@ jobs:
         -D BUILD_TESTING:BOOL=ON
         -D LAPACKE_WITH_TMG:BOOL=ON
         -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }}
+        -D LAPACK_TESTING_JUNIT_JOB_ID="${JOB_ID}"
+        ${{ startsWith(matrix.toolchain.os, 'macos') && matrix.lib_type == 'shared' && '-D USE_FLAT_NAMESPACE:BOOL=ON' || '' }}
 
     - name: Build
       run: cmake --build build -j2
@@ -137,7 +140,7 @@ jobs:
         files: build/lapack_testing_junit.xml
         # The report is named above, so there is nothing to search the tree for.
         disable_search: true
-        flags: tests-openmp-${{ matrix.toolchain.os }}-${{ matrix.lib_type }}
+        flags: tests-${{ env.JOB_ID }}
         name: openmp-${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }} (${{ matrix.lib_type }})
         fail_ci_if_error: true
         verbose: true
@@ -169,6 +172,9 @@ jobs:
       name: codecov
       deployment: false
 
+    env:
+      JOB_ID: extended-api-${{ matrix.lib_type }}
+
     strategy:
       fail-fast: false
       matrix:
@@ -190,6 +196,7 @@ jobs:
         -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }}
         -D BUILD_DEFAULT_API:BOOL=OFF
         -D BUILD_INDEX64_EXT_API:BOOL=ON
+        -D LAPACK_TESTING_JUNIT_JOB_ID="${JOB_ID}"
 
     - name: Build
       run: cmake --build build -j2
@@ -223,7 +230,7 @@ jobs:
         files: build/lapack_testing_junit.xml
         # The report is named above, so there is nothing to search the tree for.
         disable_search: true
-        flags: tests-extended-api-${{ matrix.lib_type }}
+        flags: tests-${{ env.JOB_ID }}
         name: extended-api-only (${{ matrix.lib_type }})
         fail_ci_if_error: true
         verbose: true

CMakeLists.txt

@@ -64,6 +64,9 @@ endfunction()
 
 # Use valgrind if it is found
 option( LAPACK_TESTING_USE_PYTHON "Use Python for testing. Disable it on memory checks." ON )
+set( LAPACK_TESTING_JUNIT_JOB_ID "" CACHE STRING
+  "Identifier of the job running the tests, prefixed to the JUnit report's names." )
+mark_as_advanced( LAPACK_TESTING_JUNIT_JOB_ID )
 find_program( MEMORYCHECK_COMMAND valgrind )
 if( MEMORYCHECK_COMMAND )
   message( STATUS "Found valgrind: ${MEMORYCHECK_COMMAND}" )
@@ -408,6 +411,55 @@ if(LAPACKE)
 endif()
 
 
+#-------------------------------------
+# Test summary
+#
+# The summary parses the .out files the LAPACK, BLAS and CBLAS drivers leave
+# behind, so it has to run after all of them. set_property(TEST) only reaches
+# tests added in the calling directory, hence the top level.
+if(BUILD_TESTING AND Python3_EXECUTABLE AND LAPACK_TESTING_USE_PYTHON)
+  message(STATUS "Enabling LAPACK test summary (see TESTING/testing_results.txt and lapack_testing_junit.xml)")
+
+  # Every test registered anywhere below the source root, so that adding a
+  # test directory never needs a matching edit here. Collected before the
+  # summary itself is added, so that it does not depend on itself.
+  function(_collect_tests dir out)
+    get_property(_tests DIRECTORY "${dir}" PROPERTY TESTS)
+    get_property(_subdirs DIRECTORY "${dir}" PROPERTY SUBDIRECTORIES)
+    foreach(_subdir IN LISTS _subdirs)
+      _collect_tests("${_subdir}" _subdir_tests)
+      list(APPEND _tests ${_subdir_tests})
+    endforeach()
+    set(${out} "${_tests}" PARENT_SCOPE)
+  endfunction()
+
+  _collect_tests("${LAPACK_SOURCE_DIR}" _all_tests)
+
+  # Without a job identifier the report keeps the plain suite and class names.
+  set(_summary_job_option "")
+  if(LAPACK_TESTING_JUNIT_JOB_ID)
+    set(_summary_job_option "--junit-job" "${LAPACK_TESTING_JUNIT_JOB_ID}")
+  endif()
+
+  file(COPY ${LAPACK_SOURCE_DIR}/lapack_testing.py DESTINATION ${LAPACK_BINARY_DIR})
+  # The test drivers report a numerical failure in their output and still exit
+  # with a status of 0, so this is the test that fails the run when one of them
+  # found something: --fail-on-error turns any failure or error they reported
+  # into a nonzero exit status of the summary.
+  add_test(
+    NAME LAPACK_Test_Summary
+    WORKING_DIRECTORY ${LAPACK_BINARY_DIR}
+    COMMAND ${Python3_EXECUTABLE} "lapack_testing.py"
+      "--fail-on-error" "--fail-if-empty" "--fail-on-unrecognized" "--merge-apis"
+      "--junit-xml" "lapack_testing_junit.xml" ${_summary_job_option}
+    )
+
+  if(_all_tests)
+    set_property(TEST LAPACK_Test_Summary APPEND PROPERTY DEPENDS ${_all_tests})
+  endif()
+endif()
+
+
 #-------------------------------------
 # BLAS++ / LAPACK++
 option(BLAS++ "Build BLAS++" OFF)

TESTING/CMakeLists.txt

@@ -14,19 +14,6 @@ add_subdirectory(LIN)
 add_subdirectory(EIG)
 
 
-# Only run this test if python 3 is found
-if(Python3_EXECUTABLE AND LAPACK_TESTING_USE_PYTHON)
-  message(STATUS "Enabling LAPACK test summary (see TESTING/testing_results.txt and lapack_testing_junit.xml)")
-  file(COPY ${LAPACK_SOURCE_DIR}/lapack_testing.py DESTINATION ${LAPACK_BINARY_DIR})
-  add_test(
-    NAME LAPACK_Test_Summary
-    WORKING_DIRECTORY ${LAPACK_BINARY_DIR}
-    COMMAND ${Python3_EXECUTABLE} "lapack_testing.py"
-      "--fail-if-empty" "--fail-on-unrecognized" "--merge-apis"
-      "--junit-xml" "lapack_testing_junit.xml"
-    )
-endif()
-
 function(_add_lapack_test output input target)
   set(TEST_INPUT "${LAPACK_SOURCE_DIR}/TESTING/${input}")
   set(TEST_OUTPUT "${LAPACK_BINARY_DIR}/TESTING/${output}")
@@ -39,13 +26,6 @@ function(_add_lapack_test output input target)
       -DOUTPUT=${TEST_OUTPUT}
       -DINTDIR=${CMAKE_CFG_INTDIR}
       -P "${LAPACK_SOURCE_DIR}/TESTING/runtest.cmake")
-
-    if(Python3_EXECUTABLE AND LAPACK_TESTING_USE_PYTHON)
-      set_property(
-        TEST LAPACK_Test_Summary
-        APPEND PROPERTY DEPENDS LAPACK-${testName}
-      )
-    endif()
   endif()
 endfunction()
 

lapack_testing.py

@@ -44,6 +44,11 @@
         Print only the summary table and also write a JUnit XML report
         of the analyzed output files, e.g. for GitLab CI test reports.
 
+    ./lapack_testing.py --junit-xml results.xml --junit-job linux-gfortran
+        Write a JUnit XML report whose suite and class names carry the
+        name of the job that ran the tests, so that the reports of
+        several jobs stay apart where they are collected.
+
     ./lapack_testing.py -s --markdown summary.md
         Print only the summary table and also write a GitHub-flavored
         Markdown report of the test results, e.g. for GitHub Actions
@@ -1235,7 +1240,31 @@ def output_mtime(path: Path) -> "Optional[float]":
         return None
 
 
-def junit_testcase(outcome: CaseOutcome) -> "ET.Element":
+def junit_scoped_name(job: "Optional[str]", name: str, separator: str) -> str:
+    """Prefix a JUnit name with the identifier of the job that ran it.
+
+    The same test suite is run by many CI jobs, and a report collector
+    that gathers all of them keys a test case on its suite, class and
+    test name alone.  Without the job in those names the reports of the
+    jobs collapse onto one another; with it they stay apart, and a
+    failure names the job it came from.
+
+    Args:
+        job: The job identifier passed to ``--junit-job``, or None when
+            the report is not scoped to a job.
+        name: The name to prefix.
+        separator: What to put between the two: ``"."`` for the dotted
+            class names, ``" / "`` for the free-text suite names.
+
+    Returns:
+        The prefixed name, or the name unchanged when there is no job.
+    """
+    if not job:
+        return name
+    return "{}{}{}".format(job, separator, name)
+
+
+def junit_testcase(outcome: CaseOutcome, job: "Optional[str]" = None) -> "ET.Element":
     """Build the JUnit ``<testcase>`` element of one analyzed test case.
 
     The element carries at most one status child: an ``<error>`` when
@@ -1250,6 +1279,7 @@ def junit_testcase(outcome: CaseOutcome) -> "ET.Element":
 
     Args:
         outcome: The analysis outcome of the test case.
+        job: The job identifier to prefix to the class name, or None.
 
     Returns:
         The ``<testcase>`` element.
@@ -1258,7 +1288,11 @@ def junit_testcase(outcome: CaseOutcome) -> "ET.Element":
     element = ET.Element(
         "testcase",
         {
-            "classname": "{}{}.{}".format(case.library, outcome.suffix, case.family),
+            "classname": junit_scoped_name(
+                job,
+                "{}{}.{}".format(case.library, outcome.suffix, case.family),
+                ".",
+            ),
             "name": "{} ({} {})".format(
                 case.suffixed_output(outcome.suffix),
                 PRECISION_NAMES[case.precision],
@@ -1307,7 +1341,9 @@ def junit_testcase(outcome: CaseOutcome) -> "ET.Element":
 
 
 def build_junit_tree(
-    outcomes: "Sequence[CaseOutcome]", unrecognized: "Sequence[str]"
+    outcomes: "Sequence[CaseOutcome]",
+    unrecognized: "Sequence[str]",
+    job: "Optional[str]" = None,
 ) -> "ET.ElementTree":
     """Build the JUnit XML document for the analyzed test cases.
 
@@ -1333,9 +1369,14 @@ def build_junit_tree(
     output file behind falls back to the time the report was built, so
     that the attribute is always present.
 
+    A job identifier prefixes every suite and class name, so that the
+    reports of the CI jobs that all run this one test suite do not
+    collapse onto one another where they are collected.
+
     Args:
         outcomes: The analysis outcomes, in analysis order.
         unrecognized: The names of the unrecognized ``.out`` files.
+        job: The job identifier to prefix to the names, or None.
 
     Returns:
         The document; its root is a ``<testsuites>`` element.
@@ -1359,7 +1400,7 @@ def build_junit_tree(
             root,
             "testsuite",
             {
-                "name": section_title(library, [suffix]),
+                "name": junit_scoped_name(job, section_title(library, [suffix]), " / "),
                 "timestamp": junit_timestamp(min(starts) if starts else report_time),
             },
         )
@@ -1370,7 +1411,7 @@ def build_junit_tree(
         suite_time = 0.0
         timed = False
         for outcome in suite_outcomes:
-            element = junit_testcase(outcome)
+            element = junit_testcase(outcome, job)
             suite.append(element)
             if element.find("failure") is not None:
                 failures += 1
@@ -1410,7 +1451,7 @@ def build_junit_tree(
             root,
             "testsuite",
             {
-                "name": "lapack_testing.py",
+                "name": junit_scoped_name(job, "lapack_testing.py", " / "),
                 "timestamp": junit_timestamp(report_time),
                 "tests": "1",
                 "failures": "1",
@@ -1424,7 +1465,7 @@ def build_junit_tree(
             suite,
             "testcase",
             {
-                "classname": "lapack_testing",
+                "classname": junit_scoped_name(job, "lapack_testing", "."),
                 "name": "unrecognized .out files",
                 "time": "0.000",
             },
@@ -1446,7 +1487,10 @@ def build_junit_tree(
 
 
 def write_junit_xml(
-    path: Path, outcomes: "Sequence[CaseOutcome]", unrecognized: "Sequence[str]"
+    path: Path,
+    outcomes: "Sequence[CaseOutcome]",
+    unrecognized: "Sequence[str]",
+    job: "Optional[str]" = None,
 ) -> "Optional[str]":
     """Write the JUnit XML report requested via ``--junit-xml``.
 
@@ -1459,6 +1503,7 @@ def write_junit_xml(
             are created.
         outcomes: The analysis outcomes, in analysis order.
         unrecognized: The names of the unrecognized ``.out`` files.
+        job: The job identifier to prefix to the names, or None.
 
     Returns:
         An error message if the report could not be written, otherwise
@@ -1467,7 +1512,7 @@ def write_junit_xml(
     # Path.with_name below would raise ValueError for such a path.
     if not path.name:
         return "cannot write {}: the path has no file name".format(path)
-    tree = build_junit_tree(outcomes, unrecognized)
+    tree = build_junit_tree(outcomes, unrecognized, job)
     # ET.indent is Python 3.9+; without it the report is one long line,
     # which every consumer accepts just the same.
     indent = getattr(ET, "indent", None)
@@ -1942,6 +1987,16 @@ def parse_args(argv: "Optional[Sequence[str]]" = None) -> argparse.Namespace:
         "PATH (one testcase per output file), e.g. for GitLab CI test "
         "reports; written regardless of the display and --fail-* options",
     )
+    parser.add_argument(
+        "--junit-job",
+        metavar="NAME",
+        default=None,
+        help="identifier of the job that produced the results, e.g. the "
+        "name or the Codecov flag of a CI job; it is prefixed to the suite "
+        "and class names of the --junit-xml report, so that the reports of "
+        "the jobs that all run this one test suite stay apart where they "
+        "are collected",
+    )
     parser.add_argument(
         "--markdown",
         metavar="PATH",
@@ -2306,7 +2361,9 @@ def main(argv: "Optional[Sequence[str]]" = None) -> int:
 
     junit_error: "Optional[str]" = None
     if args.junit_xml is not None:
-        junit_error = write_junit_xml(Path(args.junit_xml), outcomes, unrecognized)
+        junit_error = write_junit_xml(
+            Path(args.junit_xml), outcomes, unrecognized, args.junit_job
+        )
         if junit_error is not None:
             print("lapack_testing.py: {}".format(junit_error), file=sys.stderr)