sweep_build_configs.py

#!/usr/bin/env python3
import argparse
import subprocess
import sys
import os
import re
import shutil
import time
from collections import OrderedDict
from copy import deepcopy

DEFAULT_TARGETS_TO_BUILD = ["all"]

DEFAULT_GNU_COMPILER_VERSIONS = ["9", "8", "7", "6", "5"]
DEFAULT_MINGW_COMPILER_VERSIONS = ["5"]
DEFAULT_CLANG_COMPILER_VERSIONS = []  # ["3.8", "3.6", "3.9", "6", "8"]

DEFAULT_BUILD_CONFIGS = ["release", "debug", "release_pgo"]
DEFAULT_EASYGL_CONFIGS = ["ON", "OFF"]
DEFAULT_TATUM_EXECUTION_ENGINE_CONFIGS = ["auto", "serial"]
DEFAULT_VTR_ASSERT_LEVELS = ["4", "3", "2", "1", "0"]
DEFAULT_BLIF_EXPLORER_CONFIGS = ["ON", "OFF"]

MINGW_TOOLCHAIN_FILE = "cmake/toolchains/mingw-linux-cross-compile-to-windows.cmake"

ERROR_WARNING_REGEXES = [
    re.compile(r".*warning:.*"),
    re.compile(r".*error:.*"),
]

SUPPRESSION_ERROR_WARNING_REGEXES = [
    # We compile some .c files as C++, so we don't worry about these warnings from clang
    re.compile(r".*clang:.*warning:.*treating.*c.*as.*c\+\+.*"),
    # Don't worry about unused include directories
    re.compile(r".*clang:.*warning:.*argument.*unused.*during.*compilation:.*'(-I|-isystem)"),
]


def parse_args():
    parser = argparse.ArgumentParser(
        description="Test building VTR for multiple different compilers and build configurations"
    )

    parser.add_argument(
        "targets",
        nargs="*",
        default=DEFAULT_TARGETS_TO_BUILD,
        help="What targets to build (default: %(default)s)",
    )
    parser.add_argument(
        "-j",
        type=int,
        default=1,
        metavar="NUM_JOBS",
        help="How many parallel build jobs to allow (passed to make)",
    )
    parser.add_argument(
        "--exit_on_failure",
        action="store_true",
        default=False,
        help="Exit on first failure intead of continuing",
    )

    parser.add_argument(
        "-n",
        "--dry_run",
        action="store_true",
        default=False,
        help="Print what would happen instead of actually invoking the operations",
    )

    compiler_args = parser.add_argument_group("Compiler Configurations")
    compiler_args.add_argument(
        "--gnu_versions",
        nargs="*",
        default=DEFAULT_GNU_COMPILER_VERSIONS,
        metavar="GNU_VERSION",
        help="What versions of gcc/g++ to test (default: %(default)s)",
    )
    compiler_args.add_argument(
        "--clang_versions",
        nargs="*",
        metavar="CLANG_VERSION",
        default=DEFAULT_CLANG_COMPILER_VERSIONS,
        help="What versions of clang/clang++ to test (default: %(default)s)",
    )
    compiler_args.add_argument(
        "--mingw_versions",
        nargs="*",
        default=DEFAULT_MINGW_COMPILER_VERSIONS,
        metavar="MINGW_W64_VERSION",
        help="What versions of MinGW-W64 gcc/g++ to test for cross-compilation to Windows (default: %(default)s)",
    )

    config_args = parser.add_argument_group("Build Configurations")
    config_args.add_argument(
        "--build_types",
        nargs="*",
        default=DEFAULT_BUILD_CONFIGS,
        metavar="BUILD_TYPE",
        help="What build types to test (default: %(default)s)",
    )
    config_args.add_argument(
        "--easygl_configs",
        nargs="*",
        default=DEFAULT_EASYGL_CONFIGS,
        metavar="EASYGL_CONFIG",
        help="What EaysGL configurations to test (default: %(default)s)",
    )
    config_args.add_argument(
        "--tatum_execution_engine_configs",
        nargs="*",
        default=DEFAULT_TATUM_EXECUTION_ENGINE_CONFIGS,
        metavar="TATUM_EXECUTION_ENGINE_CONFIG",
        help="What parallel tatum configurations to test (default: %(default)s)",
    )
    config_args.add_argument(
        "--vtr_assert_levels",
        nargs="*",
        default=DEFAULT_VTR_ASSERT_LEVELS,
        metavar="VTR_ASSERT_LEVEL",
        help="What VTR assert levels to test (default: %(default)s)",
    )
    config_args.add_argument(
        "--blif_explorer_configs",
        nargs="*",
        default=DEFAULT_BLIF_EXPLORER_CONFIGS,
        metavar="BLIF_EXPLORER_CONFIG",
        help="What BlifExplorer configurations to test (default: %(default)s)",
    )

    return parser.parse_args()


def main():

    args = parse_args()

    compiler_configs = []

    for gnu_version in args.gnu_versions:
        cc = "gcc-" + gnu_version
        cxx = "g++-" + gnu_version

        config = OrderedDict()
        config["CMAKE_PARAMS"] = OrderedDict()
        config["CC"] = cc
        config["CXX"] = cxx
        compiler_configs.append(config)

    for clang_version in args.clang_versions:
        cc = "clang-" + clang_version
        cxx = "clang++-" + clang_version

        config = OrderedDict()
        config["CMAKE_PARAMS"] = OrderedDict()
        config["CC"] = cc
        config["CXX"] = cxx
        compiler_configs.append(config)

    for mingw_version in args.mingw_versions:
        config = OrderedDict()
        config["CMAKE_PARAMS"] = OrderedDict()
        config["CMAKE_PARAMS"]["CMAKE_TOOLCHAIN_FILE"] = MINGW_TOOLCHAIN_FILE
        if mingw_version != "":
            prefix = "x86_64-w64-mingw32"
            config["COMPILER_PREFIX"] = prefix
            config["MINGW_RC"] = "{}-windres".format(prefix)
            config["MINGW_CC"] = "{}-gcc-{}".format(prefix, mingw_version)
            config["MINGW_CXX"] = "{}-g++-{}".format(prefix, mingw_version)
        else:
            pass  # Use defaults
        compiler_configs.append(config)

    assert len(compiler_configs) > 0, "Must have compilers to test"

    # Take the first compiler config as the default
    default_compiler_config = compiler_configs[0]

    # The actual test compilation configurations
    test_configs = []

    # Add a default configuration test for each compiler
    for compiler_config in compiler_configs:
        test_configs.append(deepcopy(compiler_config))

    # For the default compiler, check each build type
    for build_type in args.build_types:
        config = deepcopy(default_compiler_config)
        config["BUILD_TYPE"] = build_type
        test_configs.append(config)

    # For the default compiler, check all the different build configurations
    #
    # Note that for configurations which are expected to interact (e.g. features
    # within a single tool) it may be better to test the various combinations to
    # test for bugs in non-standard configurations
    for vtr_assert_level in args.vtr_assert_levels:
        config = deepcopy(default_compiler_config)
        config["CMAKE_PARAMS"]["VTR_ASSERT_LEVEL"] = vtr_assert_level

        test_configs.append(config)

    # EasyGL is a library, so don't expect any interactions and can test seperately
    for easygl_config in args.easygl_configs:
        config = deepcopy(default_compiler_config)
        config["CMAKE_PARAMS"]["EASYGL_ENABLE_GRAPHICS"] = easygl_config

        test_configs.append(config)

    # Tatum is a library, so don't expect any interactions and can test seperately
    for tatum_execution_engine_config in args.tatum_execution_engine_configs:
        config = deepcopy(default_compiler_config)
        config["CMAKE_PARAMS"]["TATUM_EXECUTION_ENGINE"] = tatum_execution_engine_config

        test_configs.append(config)

    # Check that BlifExplorer builds with the default configuration
    for blif_explorer_config in args.blif_explorer_configs:
        config = deepcopy(default_compiler_config)
        config["CMAKE_PARAMS"]["WITH_BLIFEXPLORER"] = blif_explorer_config

        test_configs.append(config)

    # Test all the build configs
    num_failed = 0
    for config in test_configs:
        targets = args.targets
        if "CMAKE_TOOLCHAIN_FILE" in config["CMAKE_PARAMS"] and targets == DEFAULT_TARGETS_TO_BUILD:
            # Only build VPR with MINGW by default
            # The updated version of ABC (and ace/odin which depend on ABC)
            # fail to compile with MINGW
            targets = ["vpr"]

        success = build_config(args, targets, config)

        if not success:
            num_failed += 1

            if args.exit_on_failure:
                sys.exit(num_failed)

    if num_failed != 0:
        print("Failed to build {} of {} configurations".format(num_failed, len(test_configs)))
    else:
        print("Successfully built {} configurations".format(len(test_configs)))

    sys.exit(num_failed)


def build_config(args, targets, config):
    start_time = time.time()
    cc = None
    cxx = None
    build_type = None
    cmake_params = []

    # Set cc/cxx seperatley from the rest of the config parameters,
    # CC/CXX are set seperately in the command environment, rathern than
    # as command-line options
    if "CC" in config:
        cc = config["CC"]
    if "CXX" in config:
        cxx = config["CXX"]
    if "BUILD_TYPE" in config:
        build_type = config["BUILD_TYPE"]

    if "CMAKE_PARAMS" in config:
        for key, value in config["CMAKE_PARAMS"].items():
            cmake_params.append("{}={}".format(key, value))

    if not compiler_is_found(cc):
        print("Failed to find C compiler {}, skipping".format(cc))
        return False

    if not compiler_is_found(cxx):
        print("Failed to find C++ compiler {}, skipping".format(cxx))
        return False

    log_file = "build.log"

    build_successful = True
    with open(log_file, "w") as f:
        print("Building with", end=" ")
        print("Building with", end=" ", file=f)
        if cc != None:
            print(" CC={}".format(cc), end=" ")
            print(" CC={}".format(cc), end=" ", file=f)
        if cxx != None:
            print(" CXX={}:".format(cxx), end=" ")
            print(" CXX={}:".format(cxx), end=" ", file=f)
        if "CMAKE_TOOLCHAIN_FILE" in config["CMAKE_PARAMS"]:
            print(" Toolchain={}".format(config["CMAKE_PARAMS"]["CMAKE_TOOLCHAIN_FILE"]), end=" ")
            print(
                " Toolchain={}".format(config["CMAKE_PARAMS"]["CMAKE_TOOLCHAIN_FILE"]),
                end=" ",
                file=f,
            )
        print("")
        print("", file=f)
        f.flush()

        build_dir = "build"

        shutil.rmtree(build_dir, ignore_errors=True)

        os.mkdir(build_dir)

        # Copy the os environment
        new_env = {}
        new_env.update(os.environ)

        # Override CC and CXX
        if cc != None:
            new_env["CC"] = cc
        if cxx != None:
            new_env["CXX"] = cxx

        # Use the Makefile wrapper
        build_cmd = ["make"]
        build_cmd += ["BUILD_DIR={}".format(build_dir)]
        if build_type:
            build_cmd += ["BUILD_TYPE={}".format(build_type)]
        if len(cmake_params) > 0:
            build_cmd += ["CMAKE_PARAMS='{}'".format(" ".join(cmake_params))]
        build_cmd += ["-j", "{}".format(args.j)]
        build_cmd += targets

        print("  " + " ".join(build_cmd))
        print("  " + " ".join(build_cmd), file=f)
        f.flush()
        if not args.dry_run:
            try:
                subprocess.check_call(build_cmd, stdout=f, stderr=f, env=new_env)
            except subprocess.CalledProcessError as e:
                build_successful = False

    # Look for errors and warnings in the build log
    issue_count = 0
    with open(log_file) as f:
        for line in f:
            if is_valid_warning_error(line):
                print("    " + line, end=" ")
                issue_count += 1

    elapsed_time = time.time() - start_time
    if not build_successful:
        print("  ERROR: failed to compile")
    else:
        print("  OK", end=" ")
        if issue_count > 0:
            print(" ({} warnings)".format(issue_count), end=" ")
        print()
    print("  Took {:.2f} seconds".format(elapsed_time))
    return build_successful


def is_valid_warning_error(line):
    for issue_regex in ERROR_WARNING_REGEXES:
        if issue_regex.match(line):
            for suppression_regex in SUPPRESSION_ERROR_WARNING_REGEXES:
                if suppression_regex.match(line):
                    return False  # Suppressed
            return True  # Valid error/warning
    return False  # Not a problem


def compiler_is_found(execname):
    if execname == None:
        # Nothing to find
        return True

    try:
        result = subprocess.check_output([execname, "--version"], stderr=subprocess.PIPE) == 0
    except OSError as e:
        return False
    return True


if __name__ == "__main__":
    main()