diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index db0f0e54c..050f32b76 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -70,6 +70,7 @@ jobs: CC: ${{ matrix.toolchain.CC }} FFLAGS: ${{ matrix.toolchain.FFLAGS }} CFLAGS: ${{ matrix.toolchain.CFLAGS }} + JOB_ID: ${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }}-${{ matrix.lib_type }} strategy: fail-fast: false @@ -380,6 +381,7 @@ jobs: -D BUILD_TESTING:BOOL=ON \ -D LAPACKE_WITH_TMG:BOOL=ON \ -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }} \ + -D LAPACK_TESTING_JUNIT_JOB_ID="${JOB_ID}" \ "${extra[@]}" - name: Build @@ -409,7 +411,8 @@ jobs: -D LAPACKE:BOOL=ON ^ -D BUILD_TESTING:BOOL=ON ^ -D LAPACKE_WITH_TMG:BOOL=ON ^ - -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }} + -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }} ^ + -D LAPACK_TESTING_JUNIT_JOB_ID=%JOB_ID% - name: Build (Windows) if: ${{ runner.os == 'Windows' }} @@ -453,7 +456,7 @@ jobs: files: build/lapack_testing_junit.xml # The report is named above, so there is nothing to search the tree for. disable_search: true - flags: tests-${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }}-${{ matrix.lib_type }} + flags: tests-${{ env.JOB_ID }} name: ${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }} (${{ matrix.lib_type }}) fail_ci_if_error: true verbose: true diff --git a/.github/workflows/special.yml b/.github/workflows/special.yml index 0dc7bad0d..7d9698110 100644 --- a/.github/workflows/special.yml +++ b/.github/workflows/special.yml @@ -67,6 +67,7 @@ jobs: FFLAGS: "-Wall -Wno-unused-dummy-argument -Wno-unused-variable -Wno-unused-label -Werror=conversion -fimplicit-none -frecursive -fcheck=all -fopenmp" FC: ${{ matrix.toolchain.FC }} CC: ${{ matrix.toolchain.CC }} + JOB_ID: openmp-${{ matrix.toolchain.os }}-${{ matrix.lib_type }} strategy: fail-fast: false @@ -102,6 +103,8 @@ jobs: -D BUILD_TESTING:BOOL=ON -D LAPACKE_WITH_TMG:BOOL=ON -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }} + -D LAPACK_TESTING_JUNIT_JOB_ID="${JOB_ID}" + ${{ startsWith(matrix.toolchain.os, 'macos') && matrix.lib_type == 'shared' && '-D USE_FLAT_NAMESPACE:BOOL=ON' || '' }} - name: Build run: cmake --build build -j2 @@ -137,7 +140,7 @@ jobs: files: build/lapack_testing_junit.xml # The report is named above, so there is nothing to search the tree for. disable_search: true - flags: tests-openmp-${{ matrix.toolchain.os }}-${{ matrix.lib_type }} + flags: tests-${{ env.JOB_ID }} name: openmp-${{ matrix.toolchain.os }}-${{ matrix.toolchain.FC }} (${{ matrix.lib_type }}) fail_ci_if_error: true verbose: true @@ -169,6 +172,9 @@ jobs: name: codecov deployment: false + env: + JOB_ID: extended-api-${{ matrix.lib_type }} + strategy: fail-fast: false matrix: @@ -190,6 +196,7 @@ jobs: -D BUILD_SHARED_LIBS:BOOL=${{ matrix.lib_type == 'shared' && 'ON' || 'OFF' }} -D BUILD_DEFAULT_API:BOOL=OFF -D BUILD_INDEX64_EXT_API:BOOL=ON + -D LAPACK_TESTING_JUNIT_JOB_ID="${JOB_ID}" - name: Build run: cmake --build build -j2 @@ -223,7 +230,7 @@ jobs: files: build/lapack_testing_junit.xml # The report is named above, so there is nothing to search the tree for. disable_search: true - flags: tests-extended-api-${{ matrix.lib_type }} + flags: tests-${{ env.JOB_ID }} name: extended-api-only (${{ matrix.lib_type }}) fail_ci_if_error: true verbose: true diff --git a/CMakeLists.txt b/CMakeLists.txt index 72fac4209..19e40471f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,9 @@ endfunction() # Use valgrind if it is found option( LAPACK_TESTING_USE_PYTHON "Use Python for testing. Disable it on memory checks." ON ) +set( LAPACK_TESTING_JUNIT_JOB_ID "" CACHE STRING + "Identifier of the job running the tests, prefixed to the JUnit report's names." ) +mark_as_advanced( LAPACK_TESTING_JUNIT_JOB_ID ) find_program( MEMORYCHECK_COMMAND valgrind ) if( MEMORYCHECK_COMMAND ) message( STATUS "Found valgrind: ${MEMORYCHECK_COMMAND}" ) @@ -408,6 +411,55 @@ if(LAPACKE) endif() +#------------------------------------- +# Test summary +# +# The summary parses the .out files the LAPACK, BLAS and CBLAS drivers leave +# behind, so it has to run after all of them. set_property(TEST) only reaches +# tests added in the calling directory, hence the top level. +if(BUILD_TESTING AND Python3_EXECUTABLE AND LAPACK_TESTING_USE_PYTHON) + message(STATUS "Enabling LAPACK test summary (see TESTING/testing_results.txt and lapack_testing_junit.xml)") + + # Every test registered anywhere below the source root, so that adding a + # test directory never needs a matching edit here. Collected before the + # summary itself is added, so that it does not depend on itself. + function(_collect_tests dir out) + get_property(_tests DIRECTORY "${dir}" PROPERTY TESTS) + get_property(_subdirs DIRECTORY "${dir}" PROPERTY SUBDIRECTORIES) + foreach(_subdir IN LISTS _subdirs) + _collect_tests("${_subdir}" _subdir_tests) + list(APPEND _tests ${_subdir_tests}) + endforeach() + set(${out} "${_tests}" PARENT_SCOPE) + endfunction() + + _collect_tests("${LAPACK_SOURCE_DIR}" _all_tests) + + # Without a job identifier the report keeps the plain suite and class names. + set(_summary_job_option "") + if(LAPACK_TESTING_JUNIT_JOB_ID) + set(_summary_job_option "--junit-job" "${LAPACK_TESTING_JUNIT_JOB_ID}") + endif() + + file(COPY ${LAPACK_SOURCE_DIR}/lapack_testing.py DESTINATION ${LAPACK_BINARY_DIR}) + # The test drivers report a numerical failure in their output and still exit + # with a status of 0, so this is the test that fails the run when one of them + # found something: --fail-on-error turns any failure or error they reported + # into a nonzero exit status of the summary. + add_test( + NAME LAPACK_Test_Summary + WORKING_DIRECTORY ${LAPACK_BINARY_DIR} + COMMAND ${Python3_EXECUTABLE} "lapack_testing.py" + "--fail-on-error" "--fail-if-empty" "--fail-on-unrecognized" "--merge-apis" + "--junit-xml" "lapack_testing_junit.xml" ${_summary_job_option} + ) + + if(_all_tests) + set_property(TEST LAPACK_Test_Summary APPEND PROPERTY DEPENDS ${_all_tests}) + endif() +endif() + + #------------------------------------- # BLAS++ / LAPACK++ option(BLAS++ "Build BLAS++" OFF) diff --git a/TESTING/CMakeLists.txt b/TESTING/CMakeLists.txt index 89d6fa9c3..b7a78aa4c 100644 --- a/TESTING/CMakeLists.txt +++ b/TESTING/CMakeLists.txt @@ -14,19 +14,6 @@ add_subdirectory(LIN) add_subdirectory(EIG) -# Only run this test if python 3 is found -if(Python3_EXECUTABLE AND LAPACK_TESTING_USE_PYTHON) - message(STATUS "Enabling LAPACK test summary (see TESTING/testing_results.txt and lapack_testing_junit.xml)") - file(COPY ${LAPACK_SOURCE_DIR}/lapack_testing.py DESTINATION ${LAPACK_BINARY_DIR}) - add_test( - NAME LAPACK_Test_Summary - WORKING_DIRECTORY ${LAPACK_BINARY_DIR} - COMMAND ${Python3_EXECUTABLE} "lapack_testing.py" - "--fail-if-empty" "--fail-on-unrecognized" "--merge-apis" - "--junit-xml" "lapack_testing_junit.xml" - ) -endif() - function(_add_lapack_test output input target) set(TEST_INPUT "${LAPACK_SOURCE_DIR}/TESTING/${input}") set(TEST_OUTPUT "${LAPACK_BINARY_DIR}/TESTING/${output}") @@ -39,13 +26,6 @@ function(_add_lapack_test output input target) -DOUTPUT=${TEST_OUTPUT} -DINTDIR=${CMAKE_CFG_INTDIR} -P "${LAPACK_SOURCE_DIR}/TESTING/runtest.cmake") - - if(Python3_EXECUTABLE AND LAPACK_TESTING_USE_PYTHON) - set_property( - TEST LAPACK_Test_Summary - APPEND PROPERTY DEPENDS LAPACK-${testName} - ) - endif() endif() endfunction() diff --git a/lapack_testing.py b/lapack_testing.py index 6becddd0b..61bf7e3e9 100755 --- a/lapack_testing.py +++ b/lapack_testing.py @@ -44,6 +44,11 @@ Print only the summary table and also write a JUnit XML report of the analyzed output files, e.g. for GitLab CI test reports. + ./lapack_testing.py --junit-xml results.xml --junit-job linux-gfortran + Write a JUnit XML report whose suite and class names carry the + name of the job that ran the tests, so that the reports of + several jobs stay apart where they are collected. + ./lapack_testing.py -s --markdown summary.md Print only the summary table and also write a GitHub-flavored Markdown report of the test results, e.g. for GitHub Actions @@ -1235,7 +1240,31 @@ def output_mtime(path: Path) -> "Optional[float]": return None -def junit_testcase(outcome: CaseOutcome) -> "ET.Element": +def junit_scoped_name(job: "Optional[str]", name: str, separator: str) -> str: + """Prefix a JUnit name with the identifier of the job that ran it. + + The same test suite is run by many CI jobs, and a report collector + that gathers all of them keys a test case on its suite, class and + test name alone. Without the job in those names the reports of the + jobs collapse onto one another; with it they stay apart, and a + failure names the job it came from. + + Args: + job: The job identifier passed to ``--junit-job``, or None when + the report is not scoped to a job. + name: The name to prefix. + separator: What to put between the two: ``"."`` for the dotted + class names, ``" / "`` for the free-text suite names. + + Returns: + The prefixed name, or the name unchanged when there is no job. + """ + if not job: + return name + return "{}{}{}".format(job, separator, name) + + +def junit_testcase(outcome: CaseOutcome, job: "Optional[str]" = None) -> "ET.Element": """Build the JUnit ```` element of one analyzed test case. The element carries at most one status child: an ```` when @@ -1250,6 +1279,7 @@ def junit_testcase(outcome: CaseOutcome) -> "ET.Element": Args: outcome: The analysis outcome of the test case. + job: The job identifier to prefix to the class name, or None. Returns: The ```` element. @@ -1258,7 +1288,11 @@ def junit_testcase(outcome: CaseOutcome) -> "ET.Element": element = ET.Element( "testcase", { - "classname": "{}{}.{}".format(case.library, outcome.suffix, case.family), + "classname": junit_scoped_name( + job, + "{}{}.{}".format(case.library, outcome.suffix, case.family), + ".", + ), "name": "{} ({} {})".format( case.suffixed_output(outcome.suffix), PRECISION_NAMES[case.precision], @@ -1307,7 +1341,9 @@ def junit_testcase(outcome: CaseOutcome) -> "ET.Element": def build_junit_tree( - outcomes: "Sequence[CaseOutcome]", unrecognized: "Sequence[str]" + outcomes: "Sequence[CaseOutcome]", + unrecognized: "Sequence[str]", + job: "Optional[str]" = None, ) -> "ET.ElementTree": """Build the JUnit XML document for the analyzed test cases. @@ -1333,9 +1369,14 @@ def build_junit_tree( output file behind falls back to the time the report was built, so that the attribute is always present. + A job identifier prefixes every suite and class name, so that the + reports of the CI jobs that all run this one test suite do not + collapse onto one another where they are collected. + Args: outcomes: The analysis outcomes, in analysis order. unrecognized: The names of the unrecognized ``.out`` files. + job: The job identifier to prefix to the names, or None. Returns: The document; its root is a ```` element. @@ -1359,7 +1400,7 @@ def build_junit_tree( root, "testsuite", { - "name": section_title(library, [suffix]), + "name": junit_scoped_name(job, section_title(library, [suffix]), " / "), "timestamp": junit_timestamp(min(starts) if starts else report_time), }, ) @@ -1370,7 +1411,7 @@ def build_junit_tree( suite_time = 0.0 timed = False for outcome in suite_outcomes: - element = junit_testcase(outcome) + element = junit_testcase(outcome, job) suite.append(element) if element.find("failure") is not None: failures += 1 @@ -1410,7 +1451,7 @@ def build_junit_tree( root, "testsuite", { - "name": "lapack_testing.py", + "name": junit_scoped_name(job, "lapack_testing.py", " / "), "timestamp": junit_timestamp(report_time), "tests": "1", "failures": "1", @@ -1424,7 +1465,7 @@ def build_junit_tree( suite, "testcase", { - "classname": "lapack_testing", + "classname": junit_scoped_name(job, "lapack_testing", "."), "name": "unrecognized .out files", "time": "0.000", }, @@ -1446,7 +1487,10 @@ def build_junit_tree( def write_junit_xml( - path: Path, outcomes: "Sequence[CaseOutcome]", unrecognized: "Sequence[str]" + path: Path, + outcomes: "Sequence[CaseOutcome]", + unrecognized: "Sequence[str]", + job: "Optional[str]" = None, ) -> "Optional[str]": """Write the JUnit XML report requested via ``--junit-xml``. @@ -1459,6 +1503,7 @@ def write_junit_xml( are created. outcomes: The analysis outcomes, in analysis order. unrecognized: The names of the unrecognized ``.out`` files. + job: The job identifier to prefix to the names, or None. Returns: An error message if the report could not be written, otherwise @@ -1467,7 +1512,7 @@ def write_junit_xml( # Path.with_name below would raise ValueError for such a path. if not path.name: return "cannot write {}: the path has no file name".format(path) - tree = build_junit_tree(outcomes, unrecognized) + tree = build_junit_tree(outcomes, unrecognized, job) # ET.indent is Python 3.9+; without it the report is one long line, # which every consumer accepts just the same. indent = getattr(ET, "indent", None) @@ -1942,6 +1987,16 @@ def parse_args(argv: "Optional[Sequence[str]]" = None) -> argparse.Namespace: "PATH (one testcase per output file), e.g. for GitLab CI test " "reports; written regardless of the display and --fail-* options", ) + parser.add_argument( + "--junit-job", + metavar="NAME", + default=None, + help="identifier of the job that produced the results, e.g. the " + "name or the Codecov flag of a CI job; it is prefixed to the suite " + "and class names of the --junit-xml report, so that the reports of " + "the jobs that all run this one test suite stay apart where they " + "are collected", + ) parser.add_argument( "--markdown", metavar="PATH", @@ -2306,7 +2361,9 @@ def main(argv: "Optional[Sequence[str]]" = None) -> int: junit_error: "Optional[str]" = None if args.junit_xml is not None: - junit_error = write_junit_xml(Path(args.junit_xml), outcomes, unrecognized) + junit_error = write_junit_xml( + Path(args.junit_xml), outcomes, unrecognized, args.junit_job + ) if junit_error is not None: print("lapack_testing.py: {}".format(junit_error), file=sys.stderr)