Refactored the TestSet

TestSet now contains data on the success of the test in a pair

include/testharness/TestHarness.h

@@ -5,6 +5,7 @@
 #include "config/Config.h"
 #include "testharness/ResultManager.h"
 #include "tests/TestParser.h"
+#include "tests/TestResult.h"
 #include "toolchain/ToolChain.h"
 
 #include <filesystem>
@@ -17,7 +18,8 @@ namespace fs = std::filesystem;
 namespace tester {
 
 // Test hierarchy types
-typedef std::vector<std::unique_ptr<TestFile>> SubPackage;
+typedef std::pair<std::unique_ptr<TestFile>, std::reference_wrapper<std::optional<TestResult>>> TestPair;
+typedef std::vector<TestPair> SubPackage;
 typedef std::map<std::string, SubPackage> Package;
 typedef std::map<std::string, Package> TestSet;
 
@@ -54,6 +56,9 @@ class TestHarness {
 
 private:
   // The results of the tests.
+  //  NOTE we keep both a result manager and
+  //  the result in the TestSet to ensure in-ordre
+  //  printing
   ResultManager results;
 
 private:

src/analysis/Grader.cpp

@@ -125,7 +125,8 @@ void Grader::fillToolchainResultsJSON() {
         // attacker, tracking pass count.
         size_t passCount = 0, testCount = 0;
         for (const auto& subpackages : testSet[attacker]) {
-          for (const std::unique_ptr<TestFile>& test : subpackages.second) {
+          for (const TestPair& testpair : subpackages.second) {
+            const std::unique_ptr<TestFile>& test = testpair.first;
 
             TestResult result = runTest(test.get(), tc, cfg);
 
@@ -178,4 +179,4 @@ void Grader::buildResults() {
   fillToolchainResultsJSON();
 }
 
-} // End namespace tester
+} // End namespace tester

src/testharness/TestHarness.cpp

@@ -6,6 +6,7 @@
 
 #include <filesystem>
 #include <iostream>
+#include <optional>
 #include <sstream>
 #include <utility>
 
@@ -83,7 +84,7 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName)
 
       // Iterate over each test in the package
       for (size_t i = 0; i < subPackage.size(); ++i) {
-        std::unique_ptr<TestFile>& test = subPackage[i];
+        std::unique_ptr<TestFile>& test = subPackage[i].first;
         if (test->getParseError() == ParseError::NoError) {
 
           TestResult result = runTest(test.get(), toolChain, cfg);
@@ -119,8 +120,8 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName)
             << "\n";
 
   for (auto& test : invalidTests) {
-    std::cout << "  Skipped: " << test->getTestPath().filename().stem() << std::endl
-              << "  Error: " << Colors::YELLOW << test->getParseErrorMsg() << Colors::RESET << "\n";
+    std::cout << "  Skipped: " << test.first->getTestPath().filename().stem() << std::endl
+              << "  Error: " << Colors::YELLOW << test.first->getParseErrorMsg() << Colors::RESET << "\n";
   }
   std::cout << "\n";
 
@@ -150,10 +151,11 @@ void TestHarness::addTestFileToSubPackage(SubPackage& subPackage, const fs::path
 
   TestParser parser(testfile.get());
 
+  std::optional<TestResult> no_result = std::nullopt;
   if (testfile->didError()) {
-    invalidTests.push_back(std::move(testfile));
-  } else {
-    subPackage.push_back(std::move(testfile));
+    invalidTests.push_back({std::move(testfile), no_result});
+  }else {
+    subPackage.push_back({std::move(testfile), no_result});
   }
 }
 

src/tests/TestRunning.cpp

@@ -245,4 +245,4 @@ TestResult runTest(TestFile* test, const ToolChain& toolChain, const Config& cfg
   return TestResult(testPath, !testDiff, testError, "");
 }
 
-} // End namespace tester
+} // End namespace tester