Add errors for missing Testfile paths

.gitignore

@@ -1,5 +1,6 @@
 **/__pycache__/**
 venv/
+build/
 dist/
 dragon_runner.egg-info/
 scratch/

dragon_runner/config.py

@@ -11,9 +11,9 @@
 from dragon_runner.log          import log
 from dragon_runner.cli          import CLIArgs
 
-class SubPackage():
+class SubPackage(Verifiable):
     """
-    Represents a set of tests in a directory
+    Represents a set of tests in a directory.
     """
     def __init__(self, path: str): 
         self.path: str              = path
@@ -23,6 +23,12 @@ def __init__(self, path: str):
             self.tests: List[TestFile] = self.gather_tests()
         else:
             self.tests: List[TestFile] = [TestFile(path)]
+
+    def verify(self) -> ErrorCollection:
+        """
+        Verify the tests in our config have no errors.
+        """
+        return ErrorCollection(ec for test in self.tests if (ec := test.verify()))
 
     @staticmethod
     def is_test(test_path: str):
@@ -44,7 +50,7 @@ def gather_tests(self) -> List[TestFile]:
                 tests.append(TestFile(test_path))
         return tests 
 
-class Package():
+class Package(Verifiable):
     """
     Represents a single test package. Shoud have a corresponding CCID if submitted. 
     """
@@ -59,6 +65,12 @@ def __init__(self, path: str):
         else:
             self.subpackages.append(SubPackage(path))
 
+    def verify(self) -> ErrorCollection:
+        """
+        Propogate up all errors in subpackages.
+        """ 
+        return ErrorCollection(ec for spkg in self.subpackages if (ec := spkg.verify()))
+
     def add_subpackage(self, spkg: SubPackage):
         """
         Add a subpackage while keeping total test count up to date
@@ -90,13 +102,23 @@ def __init__(self, id: str, exe_path: str, runtime: str):
         self.exe_path   = exe_path 
         self.runtime    = runtime 
         self.errors     = self.verify()
-
+    
     def verify(self) -> ErrorCollection:
-        errors = ErrorCollection()
+        """
+        Check if the binary path exists and runtime path exists (if present)
+        """
+        errors = []
         if not os.path.exists(self.exe_path):
-            errors.add(ConfigError(
-                f"Cannot find binary file: {self.exe_path} in Executable: {self.id}"))
-        return errors
+            errors.append(ConfigError(
+                f"Cannot find binary file: {self.exe_path} "
+                f"in Executable: {self.id}")
+            )
+        if self.runtime and not os.path.exists(self.runtime):
+            errors.append(ConfigError(
+                f"Cannot find runtime file: {self.runtime} "
+                f"in Executable: {self.id}")
+            )
+        return ErrorCollection(errors)
 
     def source_env(self):
         """
@@ -148,8 +170,8 @@ def __init__(self, config_path: str, config_data: Dict, debug_package: Optional[
                                                    config_data.get('runtimes', ""))
         self.solution_exe       = config_data.get('solutionExecutable', None)
         self.toolchains         = self.parse_toolchains(config_data['toolchains'])
-        self.error_collection   = self.verify()
         self.packages           = self.gather_packages()
+        self.error_collection   = self.verify()
 
     def parse_executables(self, executables_data: Dict[str, str],
                                 runtimes_data: Dict[str, str]) -> List[Executable]:
@@ -200,7 +222,7 @@ def log_test_info(self):
 
     def verify(self) -> ErrorCollection:
         """
-        Assert valid paths and that all compositetoolchains and executables also verify.
+        Pass up all errrors by value in downstream objects like Toolchain, Testfile and Executable
         """
         ec = ErrorCollection()
         if not os.path.exists(self.test_dir):
@@ -209,6 +231,8 @@ def verify(self) -> ErrorCollection:
             ec.extend(exe.verify().errors)       
         for tc in self.toolchains:
             ec.extend(tc.verify().errors)
+        for pkg in self.packages:
+            ec.extend(pkg.verify().errors)
         return ec
 
     def to_dict(self) -> Dict: 

dragon_runner/errors.py

@@ -1,21 +1,46 @@
-from typing import List
+from typing import List, Union, Iterable
 
-class ConfigError:
+class Error:
+    def __str__(self): raise NotImplementedError("Must implement __str__")
+
+class ConfigError(Error):
     def __init__(self, message: str):
         self.message = message
 
     def __str__(self):
-        return f"CONFIG_ERROR: {self.message}"
+        return f"Config Error: {self.message}"
 
-class ErrorCollection:
-    def __init__(self):
-        self.errors: List[ConfigError] = []
+class TestFileError(Error):
+    def __init__(self, message: str):
+        self.message = message
 
-    def add(self, error: ConfigError):
+    def __str__(self):
+        return f"Testfile Error: {self.message}"
+
+class ErrorCollection:
+    def __init__(self, errors: Union[None, 'ErrorCollection', Iterable[Error]] = None):
+        self.errors: List[Error] = []
+        if errors is not None:
+            if isinstance(errors, ErrorCollection):
+                self.errors = errors.errors.copy()
+            elif isinstance(errors, Iterable):
+                self.errors = list(errors)
+            else:
+                raise TypeError("Must construct ErrorCollection with self or List of Error")
+
+    def has_errors(self) -> bool:
+        return self.__bool__()
+
+    def add(self, error: Error):
         self.errors.append(error)
 
-    def extend(self, errors: List[ConfigError]):
-        self.errors.extend(errors)
+    def extend(self, errors: Union['ErrorCollection', Iterable[Error]]):
+        if isinstance(errors, ErrorCollection):
+            self.errors.extend(errors.errors)
+        elif isinstance(errors, Iterable):
+            self.errors.extend(errors)
+        else:
+            raise TypeError("Must extend ErrorCollection with self or List of Error")
 
     def __bool__(self):
         return len(self.errors) > 0

dragon_runner/testfile.py

@@ -1,31 +1,55 @@
 import os
 from io                     import BytesIO
-from typing                 import Optional
+from typing                 import Optional, Union
 from dragon_runner.utils    import str_to_bytes, bytes_to_str
+from dragon_runner.errors   import Verifiable, ErrorCollection, TestFileError
 
-class TestFile:
+class TestFile(Verifiable):
     __test__ = False 
-    def __init__(self, test_path, input_dir="input",
-                                  input_stream_dir="input-stream",
-                                  output_dir="output",
-                                  comment_syntax="//"):   
+    def __init__(self, test_path, input_dir="input", input_stream_dir="input-stream",
+                                                     output_dir="output",
+                                                     comment_syntax="//"):   
         self.path                   = test_path
         self.stem, self.extension   = os.path.splitext(os.path.basename(test_path))
         self.file                   = self.stem + self.extension  
         self.input_dir              = input_dir
         self.input_stream_dir       = input_stream_dir          
         self.output_dir             = output_dir                
-        self.comment_syntax         = comment_syntax            # default C99 //
-        self.expected_out           = self.get_expected_out()   # fill expected output
-        self.input_stream           = self.get_input_stream()   # fill std input stream
-        self.expected_out_bytes     = len(self.expected_out)
-        self.input_stream_bytes     = len(self.input_stream)
-
-    def get_file_bytes(self, file_path: str) -> bytes:
-        with open(file_path, "rb") as f:
-            file_bytes = f.read()
-            assert isinstance(file_bytes, bytes), "expected bytes"
-            return file_bytes 
+        self.comment_syntax         = comment_syntax # default C99 //
+        self.verify()
+
+    def verify(self) -> ErrorCollection:
+        """
+        Ensure the paths supplied in CHECK_FILE and INPUT_FILE exist
+        """
+        collection = ErrorCollection()
+        self.expected_out           = self.get_expected_out()
+        self.input_stream           = self.get_input_stream()
+
+        # If a parse and read of a tests input or output fails, propogate here 
+        if isinstance(self.expected_out, TestFileError):
+            collection.add(self.expected_out)
+        if isinstance(self.input_stream, TestFileError):
+            collection.add(self.input_stream) 
+        if collection.has_errors():
+            return collection
+
+        self.expected_out_bytes = len(self.expected_out)
+        self.input_stream_bytes = len(self.input_stream)
+
+    def get_file_bytes(self, file_path: str) -> Optional[bytes]:
+        """
+        Get file contents in bytes
+        """
+        try:
+            with open(file_path, "rb") as f:
+                file_bytes = f.read()
+                assert isinstance(file_bytes, bytes), "expected bytes"
+                return file_bytes 
+        except FileNotFoundError:
+            return None
+        except:
+            return None
 
     def get_directive_contents(self, directive_prefix: str) -> Optional[bytes]:
         """
@@ -55,7 +79,6 @@ def get_directive_contents(self, directive_prefix: str) -> Optional[bytes]:
         contents.seek(0)
         if contents:
             content_bytes = contents.getvalue()
-            assert isinstance(content_bytes, bytes), "directive content not of type bytes"
             return content_bytes     
         return None
 
@@ -71,11 +94,10 @@ def get_file_contents(self, file_suffix, symmetric_dir) -> Optional[bytes]:
 
         same_dir_path = self.path.replace(self.extension, file_suffix)
         if os.path.exists(same_dir_path):
-            return self.get_file_bytes(same_dir_path)
-
+            return self.get_file_bytes(same_dir_path)     
         return None
 
-    def get_expected_out(self) -> bytes:
+    def get_expected_out(self) -> Union[bytes, TestFileError]:
         """
         Load the expected output for a test into a byte stream
         """
@@ -91,12 +113,13 @@ def get_expected_out(self) -> bytes:
         if check_file:
             test_dir = os.path.dirname(self.path)
             check_file_path = os.path.join(test_dir, bytes_to_str(check_file))
-            return self.get_file_bytes(check_file_path)
+            if not os.path.exists(check_file_path):
+                return TestFileError(
+                    f"Failed to locate path supplied to CHECK_FILE: {check_file_path}") 
+            return self.get_file_bytes(check_file_path) 
+        return b''# default expect empty output
 
-        # default expect empty output
-        return b''
-
-    def get_input_stream(self) -> bytes:
+    def get_input_stream(self) -> Union[bytes, TestFileError]:
         """
         Load the input stream for a test into a byte stream
         """
@@ -111,11 +134,12 @@ def get_input_stream(self) -> bytes:
         input_file = self.get_directive_contents("INPUT_FILE:")
         if input_file:
             test_dir = os.path.dirname(self.path)
-            check_file_path = os.path.join(test_dir, bytes_to_str(input_file))
-            return self.get_file_bytes(check_file_path)
-
-        # default expect empty output
-        return b''
+            input_file_path = os.path.join(test_dir, bytes_to_str(input_file))
+            if not os.path.exists(input_file_path):
+                return TestFileError(
+                    f"Failed to locate path supplied to INPUT_FILE: {input_file_path}") 
+            return self.get_file_bytes(input_file_path) 
+        return b''# default no input stream
 
     def __repr__(self):
         max_test_name_length = 30

dragon_runner/toolchain.py

@@ -62,4 +62,3 @@ def __len__(self) -> int:
 
     def __getitem__(self, index: int) -> Step:
         return self.steps[index]
-

tests/configs/gccMixConfig.json

@@ -0,0 +1,24 @@
+{
+  "testDir": "../packages/CMixedPackage",
+  "testedExecutablePaths": {
+    "gcc": "/usr/bin/gcc"
+  },
+  "toolchains": {
+    "GCC-toolchain": [
+      {
+        "stepName": "compile",
+        "executablePath": "$EXE",
+        "arguments": ["$INPUT", "-o", "$OUTPUT"],
+        "output": "/tmp/test.o",
+        "allowError": true
+      },
+      {
+        "stepName": "run",
+        "executablePath": "$INPUT",
+        "arguments": [],
+        "usesInStr": true,
+        "allowError": true
+      }
+    ]
+  }
+}

tests/configs/invalidRutnime.json

@@ -0,0 +1,27 @@
+{
+  "testDir": "../packages/CPackage",
+  "testedExecutablePaths": {
+    "gcc": "/usr/bin/gcc"
+  },
+  "runtimes": {
+    "gcc": "../lib/lib-this-runtime-dne.so"
+  },
+  "toolchains": {
+    "GCC-toolchain": [
+      {
+        "stepName": "compile",
+        "executablePath": "$EXE",
+        "arguments": ["$INPUT", "-o", "$OUTPUT"],
+        "output": "/tmp/test.o",
+        "allowError": true
+      },
+      {
+        "stepName": "run",
+        "executablePath": "$INPUT",
+        "arguments": [],
+        "usesInStr": true,
+        "allowError": true
+      }
+    ]
+  }
+}

tests/packages/CMixedPackage/mixed/000_nl.c

@@ -0,0 +1,15 @@
+#include <stdio.h>
+
+/// This will fail since there is an extra newline emitted.
+/// The expected file will have 5 bytes, the generated will have 6
+
+int main() {
+
+    printf("a\nb\nc\n");
+
+    return 0;
+}
+
+//CHECK:a
+//CHECK:b
+//CHECK:c

tests/packages/CMixedPackage/mixed/001_basic.c

@@ -0,0 +1,11 @@
+// INPUT:a
+
+#include <stdio.h>
+
+int main() {
+  char c;
+  scanf("%c", &c);
+  printf("%c", c);
+}
+
+// CHECK:a

tests/packages/CMixedPackage/mixed/001_space.c

@@ -0,0 +1,10 @@
+#include <stdio.h>
+
+int main() {
+
+    printf("print\n "); 
+    return 0;
+}
+
+// CHECK:print
+// CHECK: