From 90ff2c1366f2f22780bc43fc122d2730fbccd626 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 05:51:37 -0600
Subject: [PATCH] Refactored the TestSet

TestSet now contains data on the success of the test in a pair
---
 include/testharness/TestHarness.h |  7 ++++++-
 src/analysis/Grader.cpp           |  5 +++--
 src/testharness/TestHarness.cpp   | 14 ++++++++------
 src/tests/TestRunning.cpp         |  2 +-
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index ac32e329..d4e8e969 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -5,6 +5,7 @@
 #include "config/Config.h"
 #include "testharness/ResultManager.h"
 #include "tests/TestParser.h"
+#include "tests/TestResult.h"
 #include "toolchain/ToolChain.h"
 
 #include <filesystem>
@@ -17,7 +18,8 @@ namespace fs = std::filesystem;
 namespace tester {
 
 // Test hierarchy types
-typedef std::vector<std::unique_ptr<TestFile>> SubPackage;
+typedef std::pair<std::unique_ptr<TestFile>, std::reference_wrapper<std::optional<TestResult>>> TestPair;
+typedef std::vector<TestPair> SubPackage;
 typedef std::map<std::string, SubPackage> Package;
 typedef std::map<std::string, Package> TestSet;
 
@@ -54,6 +56,9 @@ class TestHarness {
 
 private:
   // The results of the tests.
+  //  NOTE we keep both a result manager and
+  //  the result in the TestSet to ensure in-ordre
+  //  printing
   ResultManager results;
 
 private:
diff --git a/src/analysis/Grader.cpp b/src/analysis/Grader.cpp
index b9ddcefd..1f4f561f 100644
--- a/src/analysis/Grader.cpp
+++ b/src/analysis/Grader.cpp
@@ -125,7 +125,8 @@ void Grader::fillToolchainResultsJSON() {
         // attacker, tracking pass count.
         size_t passCount = 0, testCount = 0;
         for (const auto& subpackages : testSet[attacker]) {
-          for (const std::unique_ptr<TestFile>& test : subpackages.second) {
+          for (const TestPair& testpair : subpackages.second) {
+            const std::unique_ptr<TestFile>& test = testpair.first;
 
             TestResult result = runTest(test.get(), tc, cfg);
             
@@ -178,4 +179,4 @@ void Grader::buildResults() {
   fillToolchainResultsJSON();
 }
 
-} // End namespace tester
\ No newline at end of file
+} // End namespace tester
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index a6bc59c5..d56247da 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -6,6 +6,7 @@
 
 #include <filesystem>
 #include <iostream>
+#include <optional>
 #include <sstream>
 #include <utility>
 
@@ -83,7 +84,7 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName)
 
       // Iterate over each test in the package
       for (size_t i = 0; i < subPackage.size(); ++i) {
-        std::unique_ptr<TestFile>& test = subPackage[i];
+        std::unique_ptr<TestFile>& test = subPackage[i].first;
         if (test->getParseError() == ParseError::NoError) {
         
           TestResult result = runTest(test.get(), toolChain, cfg);
@@ -119,8 +120,8 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName)
             << "\n";
 
   for (auto& test : invalidTests) {
-    std::cout << "  Skipped: " << test->getTestPath().filename().stem() << std::endl
-              << "  Error: " << Colors::YELLOW << test->getParseErrorMsg() << Colors::RESET << "\n";
+    std::cout << "  Skipped: " << test.first->getTestPath().filename().stem() << std::endl
+              << "  Error: " << Colors::YELLOW << test.first->getParseErrorMsg() << Colors::RESET << "\n";
   }
   std::cout << "\n";
 
@@ -150,10 +151,11 @@ void TestHarness::addTestFileToSubPackage(SubPackage& subPackage, const fs::path
 
   TestParser parser(testfile.get());
 
+  std::optional<TestResult> no_result = std::nullopt;
   if (testfile->didError()) {
-    invalidTests.push_back(std::move(testfile));
-  } else {
-    subPackage.push_back(std::move(testfile));
+    invalidTests.push_back({std::move(testfile), no_result});
+  }else {
+    subPackage.push_back({std::move(testfile), no_result});
   }
 }
 
diff --git a/src/tests/TestRunning.cpp b/src/tests/TestRunning.cpp
index a4ac1015..c010f412 100644
--- a/src/tests/TestRunning.cpp
+++ b/src/tests/TestRunning.cpp
@@ -245,4 +245,4 @@ TestResult runTest(TestFile* test, const ToolChain& toolChain, const Config& cfg
   return TestResult(testPath, !testDiff, testError, "");
 }
 
-} // End namespace tester
\ No newline at end of file
+} // End namespace tester