Add support for valgrind

dragon_runner/cli.py

@@ -84,9 +84,7 @@ def parse_cli_args() -> CLIArgs:
             parser.error(f"Config file is required for {args.mode} mode")
         if not os.path.isfile(args.config_file):
             parser.error(f"The config file {args.config_file} does not exist.")
-    if args.mode == "tournament" and (not bool(args.failure_log) or not bool(args.output)):
-        parser.error("Failure log and ouput file must be supplied when using tournament mode.") 
-
+
     if args.verbosity > 0:
         os.environ["DEBUG"] = str(args.verbosity)
 

dragon_runner/config.py

@@ -184,7 +184,11 @@ def find_runtime(id) -> str:
                 if rt_id == id :
                     return os.path.abspath(resolve_relative(rt_path, self.config_path))
             return ""
-        return [Executable(id, resolve_relative(path, self.config_path), find_runtime(id)) for id, path in executables_data.items()]
+        return [Executable(
+                    id,
+                    resolve_relative(path, self.config_path),
+                    find_runtime(id)
+                ) for id, path in executables_data.items()]
 
     def parse_toolchains(self, toolchains_data: Dict[str, List[Dict]]) -> List[ToolChain]:
         """

dragon_runner/harness.py

@@ -135,18 +135,18 @@ def run_tournament(self) -> bool:
         attacking_pkgs = sorted(self.config.packages, key=lambda pkg: pkg.name.lower())
         defending_exes = sorted(self.config.executables, key=lambda exe: exe.id.lower())
         solution_exe = self.config.solution_exe 
-
+        failure_log = self.cli_args.failure_log
+
         # track grader internal errors
         exit_status = True
-
-        with open(self.cli_args.failure_log, 'w') as sol_fail_log, \
-            open(self.cli_args.output_file, 'w', newline='') as grade_csv: 
-            csv_writer = csv.writer(grade_csv)
 
-            for toolchain in self.config.toolchains:
-                tc_runner = ToolChainRunner(toolchain, self.cli_args.timeout)
-                tc_table = self.create_tc_dataframe(toolchain.name, defending_exes, attacking_pkgs) 
+        for toolchain in self.config.toolchains:
+            tc_runner = ToolChainRunner(toolchain, self.cli_args.timeout)
+            tc_table = self.create_tc_dataframe(toolchain.name, defending_exes, attacking_pkgs)
+
+            with open(f"toolchain_{toolchain.name}.csv", 'w') as toolchain_csv:
                 print(f"\nToolchain: {toolchain.name}") 
+                csv_writer = csv.writer(toolchain_csv)
                 for def_exe in defending_exes: 
                     def_exe.source_env()
                     def_feedback_file = f"{def_exe.id}-{toolchain.name}feedback.txt" 
@@ -166,13 +166,15 @@ def run_tournament(self) -> bool:
                                 else:      
                                     print(Fore.RED + '.' + Fore.RESET, end='')
                                     self.log_failure_to_file(def_feedback_file, test_result)
-                                    if solution_exe == def_exe.id:
-                                        sol_fail_log.write(f"{toolchain.name} {a_pkg.name} {test.path}\n")
-                        tc_table[def_exe.id][a_pkg.name] = f"{pass_count}/{test_count}"
+                                    if solution_exe == def_exe.id and failure_log is not None:
+                                        with open(failure_log, 'a') as f_log:
+                                            f_log.write(f"{toolchain.name} {a_pkg.name} {test.path}\n")
 
+                        tc_table[def_exe.id][a_pkg.name] = f"{pass_count}/{test_count}"
+
+                # write the toolchain results into the table
                 csv_writer.writerow([toolchain.name] + [pkg.name for pkg in attacking_pkgs]) 
                 for exe in defending_exes:
                     csv_writer.writerow([exe.id] + [tc_table[exe.id][pkg.name] for pkg in attacking_pkgs])
-                csv_writer.writerow([])  # empty row for separation
 
         return exit_status 

dragon_runner/runner.py

@@ -31,9 +31,11 @@ class Command:
     """
     Wrapper for a list of arguments to run fork/exec style
     """
-    def __init__(self, args):
-        self.args: List[str]    = args
-        self.cmd: str           = self.args[0]
+    def __init__(self, args, stdout_path=None, stderr_path=None):
+        self.args: List[str]                = args
+        self.cmd: str                       = self.args[0] 
+        self.stdout_path: Optional[str]     = stdout_path
+        self.stderr_path: Optional[str]     = stderr_path
 
 @dataclass
 class CommandResult:
@@ -47,12 +49,16 @@ def log(self, level:int=0, indent=0):
         if self.subprocess:
             stdout = self.subprocess.stdout
             stderr = self.subprocess.stderr
-
-            log(f"==> {self.cmd} (exit {self.exit_status})", indent=indent, level=level)
+
+            if stderr is None:
+                stderr = b''
+            if stdout is None:
+                stdout = b''
 
+            log(f"==> {self.cmd} (exit {self.exit_status})", indent=indent, level=level)
+
             log(f"stdout ({len(stdout)} bytes):", truncated_bytes(stdout, max_bytes=512),
-                indent=indent+2, level=level)
-
+                indent=indent+2, level=level) 
             log(f"stderr ({len(stderr)} bytes):", truncated_bytes(stderr, max_bytes=512),
                 indent=indent+2, level=level)
 
@@ -125,23 +131,24 @@ def run_command(self, command: Command, stdin: bytes) -> CommandResult:
         execute a resolved command
         """
         env = os.environ.copy()
-        stdout = subprocess.PIPE
-        stderr = subprocess.PIPE
+        stdout = open(command.stdout_path, 'w') if command.stdout_path is not None else subprocess.PIPE
+        stderr = open(command.stderr_path, 'w') if command.stderr_path is not None else subprocess.PIPE
+
         start_time = time.time()
         cr = CommandResult(cmd=command.cmd)
         try:
             result = subprocess.run(command.args, env=env, input=stdin, stdout=stdout,
                                     stderr=stderr, check=False, timeout=self.timeout)
             wall_time = time.time() - start_time
-            cr.subprocess=result
-            cr.exit_status=result.returncode
-            cr.time=wall_time
+            cr.subprocess = result
+            cr.exit_status = result.returncode
+            cr.time = wall_time
         except TimeoutExpired:
-            cr.time=0
-            cr.timed_out=True
-            cr.exit_status=255 
+            cr.time = self.timeout # max time
+            cr.timed_out = True
+            cr.exit_status = 255 
         except Exception:
-            cr.exit_status=1
+            cr.exit_status = 1
 
         return cr
 
@@ -157,7 +164,11 @@ def resolve_command(self, step: Step, params: MagicParams) -> Command:
         """
         replace magic parameters with real arguments
         """
-        command = Command([step.exe_path] + step.arguments)
+        command = Command(
+            args=[step.exe_path] + step.arguments,
+            stdout_path = step.stdout_path,
+            stderr_path = step.stderr_path
+        )
         command = self.replace_magic_args(command, params)
         command = self.replace_env_vars(command)
         exe = command.args[0]
@@ -287,24 +298,27 @@ def replace_env_vars(cmd: Command) -> Command:
                 resolved.append(arg)
             else:
                 resolved.append(arg)
-        return Command(resolved)
+        cmd.args = resolved 
+        return cmd
 
     @staticmethod
-    def replace_magic_args(command: Command, params: MagicParams) -> Command:
+    def replace_magic_args(command: Command, params: MagicParams) -> Command: 
         """
         Magic args are inherited from previous steps
         """
         resolved = []
         for arg in command.args:
-            if arg == '$EXE':
-                resolved.append(params.exe_path)
-            elif arg == '$INPUT':
-                resolved.append(params.input_file)
-            elif arg == '$OUTPUT':
-                resolved.append(params.output_file) 
+            if '$EXE' in arg:
+                resolved.append(arg.replace('$EXE', params.exe_path))
+            elif '$INPUT' in arg and params.input_file:
+                resolved.append(arg.replace('$INPUT', params.input_file))
+            elif '$OUTPUT' in arg and params.output_file:
+                resolved.append(arg.replace('$OUTPUT', params.output_file))
             else:
                 resolved.append(arg)
-        return Command(resolved)
+        command.args = resolved
+        command.cmd = command.args[0]
+        return command
 
 def diff_bytes(s1: bytes, s2: bytes) -> str:
     """

dragon_runner/scripts/grade.py

@@ -9,7 +9,6 @@
 import csv
 from pathlib    import Path 
 from fractions  import Fraction
-from typing     import List
 
 # These are hard coded as to not bloat the CLI. Probably easier to change in place.
 DEFENSIVE_PTS       = 2
@@ -33,36 +32,26 @@ def normalize_competetive_scores(tc_table):
     Normalize the competative scores of a table relative to the max score.
     By convention the last row contains the total score for the toolchain  
     """
-    # n_rows = len(tc_table)
-    # n_cols = len(tc_table[0])    
-
-    print("TC_TABLE", tc_table)
-    print("COMPETITIVE ROW:", tc_table[-2][1:])
-    raw_competitive_scores = tc_table[-2][1:]
+    raw_competitive_scores = [float(score) for score in tc_table[-2][1:]]
     max_score = max(raw_competitive_scores)
-    print("MAX SCORE: ", max_score)
+    print("MAX SCORE: ", max_score, "FROM :", raw_competitive_scores)
     norm_competitive_scores = [
         round(COMPETITIVE_WEIGHT * (float(score) / float(max_score)), 3)
         for score in raw_competitive_scores
     ]
     norm_scores_row = ["Normalized Points (20% Weight)"] + norm_competitive_scores
     tc_table.append(norm_scores_row)
 
-def average_toolchain_tables(tables):
+def average_toolchain_tables(tables, n_compilers):
     """
     Take a number of identical toolchain tables and return the average
     of all their values. 
     """ 
-    n_rows = len(tables[0])
-    n_cols = len(tables[0][0])
-    for table in tables:
-        assert len(table) == n_rows, "num rows differ"
-        assert len(table[0]) == n_cols, "num cols differ"
-
+    print("N_COMPILERS: ", n_compilers)
     avg_table = [row[:] for row in tables[0]]
     avg_table[0][0] = "toolchain summary" 
-    for i in range(1, n_cols):
-        for j in range(1, n_cols):
+    for i in range(1, n_compilers+1):
+        for j in range(1, n_compilers+1):
             avg = 0 
             for tc in tables:
                 avg += to_float(tc[i][j])
@@ -76,29 +65,28 @@ def add_competitive_rows(table):
     Add a row at the bottom of the table for defensive, offesnsive
     and coherence points. Not yet normalized to highest score. 
     """
-    n_cols = len(table[0])
-    print("N_COLS:", n_cols)
-    print("N_ROWS:", len(table))
-    
+    n_compilers = len(table)-1 # one row for labels 
+    print(table)
+    print("N_COMPILERS: ", n_compilers)
+
     # Declare new rows
-    ta_points_row = ["TA Testing Score (50% Weight)"] + [0] * (n_cols - 1)
-    defensive_row = ["Defensive Points"] + [0] * (n_cols - 1)
-    offensive_row = ["Offensive Points"] + [0] * (n_cols - 1)
-    coherence_row = ["Coherence Points"] + [0] * (n_cols - 1)
-    total_row = ["Competitive Points"] + [0] * (n_cols - 1)
-
-    for j in range(1, n_cols):
-        attacker = table[0][j]
+    ta_points_row = ["TA Testing Score (50% Weight)"] + [0] * (n_compilers)
+    defensive_row = ["Defensive Points"] + [0] * (n_compilers)
+    offensive_row = ["Offensive Points"] + [0] * (n_compilers)
+    coherence_row = ["Coherence Points"] + [0] * (n_compilers)
+    total_row = ["Competitive Points"] + [0] * (n_compilers)
+
+    for j in range(1, n_compilers+1):
+
         ta_score = 0 # score on "solution" package
         o_score = 0 # offensive score
         d_score = 0 # defensive score
         c_score = 0 # coherence score
         c_score += COHERENCE_PTS if to_float(table[j][j]) == 1 else 0
 
         # defender = table[1][0]
-        for i in range(1, n_cols):
+        for i in range(1, n_compilers+1):
             defender = table[i][0]
-            print(f"i: {i}", defender)
             if defender == SOLUTION:
                 # look at the transpose position to determine TA score
                 ta_score += to_float(table[j][i])
@@ -111,8 +99,8 @@ def add_competitive_rows(table):
         defensive_row[j] = str(round(d_score, 2))
         offensive_row[j] = str(round(o_score, 2))
         coherence_row[j] = round(c_score, 3)
-        total_row[j] = str(
-                float(defensive_row[j]) + float(offensive_row[j]) + float(coherence_row[j]))
+        total_row[j] = str(round(
+                float(defensive_row[j]) + float(offensive_row[j]) + float(coherence_row[j]), 3))
 
     # Append new rows to the table
     table.append(defensive_row)
@@ -123,24 +111,34 @@ def add_competitive_rows(table):
 
     return table
 
-def tournament(tournament_csv_paths: List[str], grade_path: str):
+def grade(*args):
     """
     Run the tournament for each tournament csv then average all the
     toolchain tables. Write all the tables including the average to 
     the final grade_path
     """
+
+    if len(args) < 2:
+        print("Must supply at least two arguments: <toolchain_csv>+ <grade_csv>")
+        return 1
+
+    # do some mangaling with args to be able to pass in variadic from loader.py
+    toolchain_csv_paths = args[:-1]
+    grade_path = args[-1]
+    n_compilers = 0
+
     tc_tables = [] 
-    for file in tournament_csv_paths:
+    for file in toolchain_csv_paths:
         with open(file, 'r') as f:
             reader = csv.reader(f)
             tc_table = list(reader)
             tc_tables.append(add_competitive_rows(tc_table))
 
-    print(tc_tables)
-    tc_avg = average_toolchain_tables(tc_tables)
+    n_compilers = len(tc_tables[0][0]) - 1
+    print("N COMPILERS: ", n_compilers)
+    tc_avg = average_toolchain_tables(tc_tables, n_compilers)
     normalize_competetive_scores(tc_avg)
-    print(tc_avg)
-
+
     with open(grade_path, 'w') as f:
         writer = csv.writer(f)
         for table in tc_tables:
@@ -164,28 +162,5 @@ def tournament(tournament_csv_paths: List[str], grade_path: str):
     )
 
     args = parser.parse_args() 
-    tournament(args.tournament_csvs, args.log_file)
+    grade(*args.tournament_csvs, args.output_csv)
 
-    #
-    # input_files = ['Grades.csv'] 
-    # tc_tables = [] 
-    # for file in input_files:
-    #     with open(file, 'r') as f:
-    #         reader = csv.reader(f)
-    #         tc_table = list(reader)
-    #         tc_tables.append(add_competitive_rows(tc_table))
-    # 
-    # print(tc_tables)
-    # tc_avg = average_toolchain_tables(tc_tables)
-    # normalize_competetive_scores(tc_avg)
-    # print(tc_avg)
-    #
-    # output_file = './vcalc-grades.csv'
-    # with open(output_file, 'w') as f:
-    #     writer = csv.writer(f)
-    #     for table in tc_tables:
-    #         writer.writerows(table)
-    #         writer.writerow([]) # newline
-    #     writer.writerows(tc_avg)
-
-

dragon_runner/scripts/loader.py

@@ -1,6 +1,7 @@
 
 from typing import List
 from dragon_runner.scripts.build import build 
+from dragon_runner.scripts.grade import grade 
 from dragon_runner.scripts.gather import gather 
 from dragon_runner.scripts.gen_config import main as gen_config 
 
@@ -24,6 +25,7 @@ def unknown_script():
 
         script_dispatch = {
             "build":        lambda: build(*self.args),
+            "grade":        lambda: grade(*self.args),
             "gather":       lambda: gather(*self.args),
             "gen-config":   lambda: gen_config(*self.args),
             "anon-tests":   lambda: print("TODO"),