From 48945758bfcbe08966363c49377f04a1a361d4b0 Mon Sep 17 00:00:00 2001 From: Ayrton Chilibeck Date: Sun, 8 Sep 2024 07:02:05 -0600 Subject: [PATCH] Seperation of test running and result printing --- include/testharness/TestHarness.h | 2 + src/testharness/TestHarness.cpp | 116 +++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h index 3c2d207..a77b2b1 100644 --- a/include/testharness/TestHarness.h +++ b/include/testharness/TestHarness.h @@ -64,9 +64,11 @@ class TestHarness { private: // test running bool runTestsForToolChain(std::string tcId, std::string exeName); + void threadRunTestsForToolChain(std::string tcId, std::string exeName); // helper for formatting tester output void printTestResult(const TestFile *test, TestResult result); + bool aggregateTestResultsForToolChain(std::string tcName, std::string exeName); // test finding and filling methods void addTestFileToSubPackage(SubPackage& subPackage, const fs::path& file); diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp index c702d4f..05ccebc 100644 --- a/src/testharness/TestHarness.cpp +++ b/src/testharness/TestHarness.cpp @@ -4,10 +4,12 @@ #include "tests/TestRunning.h" #include "util.h" +#include #include #include #include #include +#include #include namespace tester { @@ -23,8 +25,11 @@ bool TestHarness::runTests() { for (auto exePair : cfg.getExecutables()) { // Iterate over toolchains. for (auto& tcPair : cfg.getToolChains()) { - if (runTestsForToolChain(exePair.first, tcPair.first) == 1) + std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first); + if (aggregateTestResultsForToolChain(tcPair.first, exePair.first) == 1) failed = true; + + t.join(); } } return failed; @@ -59,6 +64,115 @@ void TestHarness::printTestResult(const TestFile *test, TestResult result) { std::cout << "\n"; } +bool TestHarness::aggregateTestResultsForToolChain(std::string tcName, std::string exeName) { + bool failed = false; + + ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use. + const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested. + toolChain.setTestedExecutable(exe); + + if (cfg.hasRuntime(exeName)) // If we have a runtime, set that as well. + toolChain.setTestedRuntime(cfg.getRuntimePath(exeName)); + else + toolChain.setTestedRuntime(""); + + std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n'; + std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n'; + + unsigned int toolChainCount = 0, toolChainPasses = 0; // Stat tracking for toolchain tests. + + // Iterate over each package. + for (auto& [packageName, package] : testSet) { + std::cout << "Entering package: " << packageName << '\n'; + unsigned int packageCount = 0, packagePasses = 0; + + // Iterate over each subpackage + for (auto& [subPackageName, subPackage] : package) { + std::cout << " Entering subpackage: " << subPackageName << '\n'; + unsigned int subPackagePasses = 0, subPackageSize = subPackage.size(); + + // Iterate over each test in the package + for (size_t i = 0; i < subPackage.size(); ++i) { + TestPair& pair = subPackage[i]; + std::unique_ptr& test = pair.first; + if (test->getParseError() == ParseError::NoError) { + + // Poll while we wait for the result + // TODO this could probably be replaced with some sort of interrupt, + // (and probably should be), but better this than no threads + while (!pair.second.has_value()) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + TestResult result = pair.second.value(); + + // keep the result with the test for pretty printing + std::optional res_clone = std::make_optional(result.clone()); + subPackage[i].second.swap(res_clone); + + results.addResult(exeName, tcName, subPackageName, result); + printTestResult(test.get(), result); + + if (result.pass) { + ++packagePasses; + ++subPackagePasses; + } else { + failed = true; + } + } else { + std::cout << " " << (Colors::YELLOW + "[INVALID]" + Colors::RESET) << " " + << test->getTestPath().stem().string() << '\n'; + --subPackageSize; + } + } + std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize << '\n'; + // Track how many tests we run. + packageCount += subPackageSize; + } + + // Update the toolchain stats from the package stats. + toolChainPasses += packagePasses; + toolChainCount += packageCount; + + std::cout << " Package passed " << packagePasses << " / " << packageCount << '\n'; + } + + std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n"; + std::cout << "Invalid " << invalidTests.size() << " / " << toolChainCount + invalidTests.size() + << "\n"; + + for (auto& test : invalidTests) { + std::cout << " Skipped: " << test.first->getTestPath().filename().stem() << std::endl + << " Error: " << Colors::YELLOW << test.first->getParseErrorMsg() << Colors::RESET << "\n"; + } + std::cout << "\n"; + + return failed; +} + +void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exeName) { + ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use. + const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested. + toolChain.setTestedExecutable(exe); + + // Iterate over each package. + for (auto& [packageName, package] : testSet) { + // Iterate over each subpackage + for (auto& [subPackageName, subPackage] : package) { + // Iterate over each test in the package + for (size_t i = 0; i < subPackage.size(); ++i) { + std::unique_ptr& test = subPackage[i].first; + if (test->getParseError() == ParseError::NoError) { + + TestResult result = runTest(test.get(), toolChain, cfg); + // keep the result with the test for pretty printing + std::optional res_clone = std::make_optional(result.clone()); + subPackage[i].second.swap(res_clone); + } + } + } + } +} + bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) { bool failed = false;