diff --git a/dragon_runner/runner.py b/dragon_runner/runner.py index 0642900..0e959ca 100644 --- a/dragon_runner/runner.py +++ b/dragon_runner/runner.py @@ -15,7 +15,8 @@ from dragon_runner.toolchain import Step from dragon_runner.cli import CLIArgs from dragon_runner.utils import make_tmp_file, bytes_to_str,\ - file_to_bytes, truncated_bytes + file_to_bytes, truncated_bytes, \ + set_timeout init(autoreset=True) @@ -218,7 +219,6 @@ def run(self, test: TestFile, exe: Executable) -> Optional[TestResult]: """ timeout_msg = f"Toolchain timed out for test: {test.file}" tr.did_pass=False; - # tr.did_panic=True; tr.failing_step=step.name; tr.error_msg=timeout_msg; return tr @@ -244,9 +244,14 @@ def run(self, test: TestFile, exe: Executable) -> Optional[TestResult]: error_pattern = r'.*?(Error on line \d+):?.*' else: error_pattern = r'\s*(\w+Error):?.*' - if lenient_diff(step_stderr, expected, error_pattern) == "": - tr.did_pass = True - else: + + # Check if diffs timeout eventually on massive stderrr dumps + try: + if lenient_diff(step_stderr, expected, error_pattern) == "": + tr.did_pass = True + else: + tr.did_pass = False + except TimeoutError: tr.did_pass = False return tr; @@ -352,12 +357,13 @@ def precise_diff(produced: bytes, expected: bytes) -> str: return "" return diff_bytes(produced, expected) +@set_timeout(timeout=5) def lenient_diff(produced: bytes, expected: bytes, pattern: str) -> str: """ Perform a lenient diff on error messages, using the pattern as a mask/filter. Unfortunately we have to convert from and back to bytes in order to apply regex. Bytes must be UTF-8 decodable. - """ + """ produced_str = p.strip() if (p := bytes_to_str(produced)) else None expected_str = e.strip() if (e := bytes_to_str(expected)) else None diff --git a/dragon_runner/utils.py b/dragon_runner/utils.py index 804edeb..3f131d7 100644 --- a/dragon_runner/utils.py +++ b/dragon_runner/utils.py @@ -1,9 +1,9 @@ import os import sys +import multiprocessing import tempfile from typing import Optional from colorama import init - # Initialize colorama init(autoreset=True) @@ -111,3 +111,25 @@ def bytes_to_file(file: str, data: bytes) -> Optional[str]: print(f"Writting bytes to file failed with: {e}") return None +def set_timeout(timeout): + """ + Decorator to make a function throw a timeout error if it is taking too + long to execute. + """ + def decorator(func): + def wrapper(*args, **kwargs): + result = multiprocessing.Queue() + process = multiprocessing.Process( + target=lambda q: q.put(func(*args, **kwargs)), args=(result,)) + process.start() + process.join(timeout) + if process.is_alive(): + process.terminate() + raise TimeoutError(f"Function '{func.__name__}' timed out after {timeout} seconds") + outcome = result.get() + if isinstance(outcome, Exception): + raise outcome + return outcome + return wrapper + return decorator + diff --git a/tests/conftest.py b/tests/conftest.py index 3a5ca3e..21465b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ def create_cli_args(**kwargs) -> CLIArgs: failure_log = kwargs.get('failure_log', None), debug_package = kwargs.get('debug_package', None), mode = kwargs.get('mode', None), - timeout = kwargs.get('timeout', None), + timeout = kwargs.get('timeout', 5), time = kwargs.get('time', None), verbosity = kwargs.get('verbosity', None), verify = kwargs.get('verify', None),