Create all ephemeral files in .test-artifacts

include/toolchain/Command.h

@@ -32,7 +32,7 @@ class Command {
   Command(const Command& command) = default;
 
   // Destructor for removing temporary files
-  ~Command() {}
+  ~Command();
 
   // Execute the command.
   ExecutionOutput execute(const ExecutionInput& ei) const;

include/toolchain/ExecutionState.h

@@ -1,6 +1,7 @@
 #ifndef TESTER_EXECUTION_STATE_H
 #define TESTER_EXECUTION_STATE_H
 
+#include <iostream>
 #include <filesystem>
 #include <optional>
 namespace fs = std::filesystem;
@@ -21,6 +22,10 @@ class ExecutionInput {
         testedExecutable(std::move(testedExecutable)),
         testedRuntime(std::move(testedRuntime)) {}
 
+  ~ExecutionInput() {
+    // std::cout << "Destory execution input: " << inputPath << std::endl;
+  }
+
   // Gets input file.
   const fs::path& getInputFile() const { return inputPath; }
 
@@ -52,6 +57,10 @@ class ExecutionOutput {
         errPath(std::move(errPath)), 
         rv(0), elapsedTime(0), hasElapsed(false), isErrorTest(false) {}
 
+  ~ExecutionOutput() {
+    // std::cout << "Destory execution output: " << outPath << std::endl;
+  }
+
   // Gets output file.
   fs::path getOutputFile() const { return outPath; }
   fs::path getErrorFile() const { return errPath; }

src/main.cpp

@@ -34,7 +34,7 @@ int main(int argc, char** argv) {
 
     // Free resources
   } catch (const std::runtime_error& e) {
-    std::cout << "Test harness error: " << e.what() << '\n';
+    std::cout << e.what() << '\n';
     return 1;
   }
 

src/tests/TestFile.cpp

@@ -25,37 +25,48 @@ uint64_t TestFile::generateId() {
 TestFile::TestFile(const fs::path& path, const fs::path& artifactDir)
     : id(generateId()), testPath(path) {
 
-  std::string testName = path.stem();
-  setInsPath(artifactDir / fs::path(testName + '-' + std::to_string(id) + ".ins"));
-  setOutPath(artifactDir / fs::path(testName + '-' + std::to_string(id) + ".out"));
-
-  std::cout << "Creating file: " << testName << std::endl;
-  std::cout << "INS: " << getInsPath() << std::endl;  
-  std::cout << "OUT: " << getInsPath() << std::endl;  
-
   try {
-    // Create tmp directory if it doesn't exist
+    // Create .test-artifacts if it doesn't exist    
     if (!fs::exists(artifactDir)) {
-      if (!fs::create_directories(artifactDir)) {
-        throw std::runtime_error("Failed to create directory: " + artifactDir.string());
-      }
+      fs::create_directories(artifactDir);
+    }
+
+    // create .test-artifacts/testfiles if it doesn't exist
+    fs::path testArtifactsDir = artifactDir / "testfiles"; 
+    if (!fs::exists(testArtifactsDir)) {
+      fs::create_directories(testArtifactsDir);
     }
+
+    std::string testName = path.stem();
+    fs::path basePath = testArtifactsDir / fs::path(testName + '-' + std::to_string(id));
+
+    setInsPath(fs::path(basePath.string() + ".ins"));
+    setOutPath(fs::path(basePath.string() + ".out"));
+
+    // std::cout << "Creating file: " << testName << std::endl;
+    // std::cout << "INS: " << getInsPath() << std::endl;  
+    // std::cout << "OUT: " << getOutPath() << std::endl;  
+
   } catch (const fs::filesystem_error& e) {
     throw std::runtime_error("Filesystem error: " + std::string(e.what()));
   } 
 }
 
 TestFile::~TestFile() {
 
-  std::cout << "Calling Destructor...\n";
-  // if (fs::exists(insPath)) {
-  //   // Remove temporary input stream file 
-  //   fs::remove(insPath);
-  // }
-  // if (fs::exists(outPath)) {
-  //   // Remove the tenmporary testfile directory and the expected out
-  //   fs::remove(outPath);
-  // }
+  // std::cout << "Calling Destructor...\n";
+  try {
+    if (fs::exists(insPath)) {
+      // Remove temporary input stream file 
+      fs::remove(insPath);
+    }
+    if (fs::exists(outPath)) {
+      // Remove the tenmporary testfile directory and the expected out
+      fs::remove(outPath);
+    }
+  } catch (const std::exception& e) {
+    std::cerr << "Caught exception in destructor: "<< e.what() << std::endl;
+  }
 }
 
 std::string TestFile::getParseErrorMsg() const {

src/tests/TestParser.cpp

@@ -35,22 +35,14 @@ ParseError copyFile(const fs::path& from, const fs::path& to) {
 } 
 
 void TestParser::insLineToFile(fs::path filePath, std::string line, bool firstInsert) {
-
-  // Set the mode to open the file determined by first insert
-  std::ios_base::openmode mode;
+
   if (firstInsert) {
-    mode = std::ios::out | std::ios::trunc;
+    std::ofstream out(filePath);
+    out << line;
   } else {
-    mode = std::ios::app;
-  }
-
-  // Open the file, write the contents.
-  std::ofstream out(filePath, mode);
-  out << line;
-  if (!firstInsert) {
-      out << "\n";
-  }
-  out.close();
+    std::ofstream out(filePath, std::ios::app);
+    out << "\n" << line;
+  }   
 }
 
 /**
@@ -90,7 +82,7 @@ ParseError TestParser::matchInputDirective(std::string& line) {
 
   size_t findIdx = line.find(Directive::INPUT);
   std::string inputLine = line.substr(findIdx + Directive::INPUT.length());
-  
+
   insLineToFile(testfile->getInsPath(), inputLine, !foundInput);
   foundInput = true;
 
@@ -110,7 +102,7 @@ ParseError TestParser::matchCheckDirective(std::string& line) {
 
   size_t findIdx = line.find(Directive::CHECK);
   std::string checkLine = line.substr(findIdx + Directive::CHECK.length());
-  
+
   insLineToFile(testfile->getOutPath(), checkLine, !foundCheck);
   foundCheck = true;
 
@@ -134,6 +126,7 @@ ParseError TestParser::matchInputFileDirective(std::string& line) {
     auto inputPath = std::get<fs::path>(path);
     copyFile(inputPath, testfile->getInsPath());
     foundInputFile = true;
+    return ParseError::NoError;
   }
 
   return std::get<ParseError>(path);

src/tests/TestRunning.cpp

@@ -149,10 +149,21 @@ void formatFileDump(const fs::path& testPath, const fs::path& expOutPath,
                     const fs::path& genOutPath) {
   std::cout << "----- TestFile: "<< testPath.filename() << std::endl;
   dumpFile(testPath);
-  std::cout << "----- Expected Output (" << fs::file_size(expOutPath) << " bytes)" << std::endl;
-  dumpFile(expOutPath, true);
-  std::cout << "----- Generated Output (" << fs::file_size(genOutPath) << " bytes)" << std::endl;
-  dumpFile(genOutPath, true);
+
+  if (fs::exists(expOutPath)) {
+    std::cout << "----- Expected Output (" << fs::file_size(expOutPath) << " bytes)" << std::endl;
+    dumpFile(expOutPath, true);
+  } else {
+    std::cout << "----- Expected Output: (0 bytes)" << std::endl;
+    dumpFile("/dev/null", true);
+  } 
+  if (fs::exists(genOutPath)) {
+    std::cout << "----- Generated Output (" << fs::file_size(genOutPath) << " bytes)" << std::endl;
+    dumpFile(genOutPath, true);
+  } else {
+    std::cout << "----- Generated Output: (0 bytes)" << std::endl;
+    dumpFile("/dev/null", true);
+  } 
   std::cout << "-----------------------" << std::endl;
 }
 

src/toolchain/Command.cpp

@@ -22,7 +22,7 @@ namespace {
 /// @brief Open the file with provided flags and mode. Redirect the file descriptor
 /// supplied by dup_fd to the file underlying file_str. 
 int redirectStdStream(const std::string& file_str, int flags, mode_t mode, int dup_fd) {
-
+   
   // Open the process
   int fd = open(file_str.c_str(), flags, mode);
   if (fd == -1) {
@@ -78,13 +78,19 @@ void becomeCommand(const std::string& exe,
 
   // Open the supplied files and redirect FD of the current child process to them.
   int outFileStatus = redirectStdStream(output.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR, STDOUT_FILENO);
-  int errorFileStatus = redirectStdStream(error.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR, STDERR_FILENO);
-  int inFileStatus = !input.empty() ? redirectStdStream(input.c_str(), O_RDONLY, 0, STDIN_FILENO) : 0;
+  int errFileStatus = redirectStdStream(error.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR, STDERR_FILENO);
+  int insFileStatus = !input.empty() ? redirectStdStream(input.c_str(), O_RDONLY, 0, STDIN_FILENO) : 0;
 
   // If opening any of the supplied output, input, or error files failed, raise here.
-  if (outFileStatus == -1 || errorFileStatus == -1 || inFileStatus == -1) {
-      perror("dup2");
-      exit(EXIT_FAILURE);
+  if (outFileStatus == -1) {
+    perror("dup2 failed to open output file");
+    exit(EXIT_FAILURE);
+  } else if (errFileStatus == -1) {
+    perror("dup2 failed to open error file");
+    exit(EXIT_FAILURE);
+  } else if (insFileStatus == -1) {
+    perror("dup2 failed to open input stream file");
+    exit(EXIT_FAILURE);
   }
 
   // Replace ourselves with the command.
@@ -105,11 +111,13 @@ void runCommand(std::promise<unsigned int>& promise, std::atomic_bool& killVar,
                 const std::string& output,
                 const std::string& error,
                 const std::string& runtime) {
-
+
+#if defined(DEBUG) 
   std::cerr << "Running Command: " << exe << std::endl; 
-  std::cerr << "Made out file: " << input << std::endl;
-  std::cerr << "Made error file: " << output << std::endl;
-
+  std::cerr << "Made out file: " << output << std::endl;
+  std::cerr << "Made error file: " << error << std::endl;
+  std::cerr << "Made input file: " << input << std::endl;
+#endif 
   pid_t childId = fork();
 
   // We're the child process, we want to replace our process image with the
@@ -179,7 +187,7 @@ namespace tester {
 Command::Command(const JSON& step, int64_t timeout)
     : usesRuntime(false), usesInStr(false), timeout(timeout) {
 
-  fs::path tmpPath = "./tmp";
+  fs::path testArtifactsPath = "./.test-artifacts";
 
   // Make sure the step has all of the values needed for construction.
   ensureContains(step, "stepName");
@@ -197,17 +205,16 @@ Command::Command(const JSON& step, int64_t timeout)
 
   // Allow override of stdout path with output property
   if (doesContain(step, "output")) {
-    outPath = tmpPath / fs::path(step["output"]);
+    outPath = testArtifactsPath / fs::path(step["output"]);
   } else {
-    std::string output_name = std::string(step["stepName"]) + ".stdout";
-    outPath = tmpPath / output_name;
+    std::string outFileName = std::string(step["stepName"]) + ".stdout";
+    outPath = testArtifactsPath / outFileName;
   }
 
-  std::cout  << "Created commadn with outpath: " << outPath << std::endl; 
   // Always create a stderr path
-  std::string error_name = std::string(step["stepName"]) + ".stderr";
-  errPath = tmpPath / error_name;
-
+  std::string errFileName = std::string(step["stepName"]) + ".stderr";
+  errPath = testArtifactsPath / fs::path(errFileName);
+   
   // Do we use an input stream file?
   if (doesContain(step, "usesInStr"))
     usesInStr = step["usesInStr"];
@@ -218,10 +225,18 @@ Command::Command(const JSON& step, int64_t timeout)
 
   // Do we allow errors?
   if (doesContain(step, "allowError"))
-    allowError = step["allowError"]; 
+    allowError = step["allowError"];
+
+  // std::cout  << "Created command with outpath: " << outPath << std::endl; 
+  // std::cout  << "Created command with errPath: " << errPath << std::endl;  
+}
+
+Command::~Command() {
+  // std::cout << "Destroying command\n";
 }
 
 ExecutionOutput Command::execute(const ExecutionInput& ei) const {
+
   // Create our output context.
   ExecutionOutput eo(outPath, errPath);
 
@@ -235,11 +250,17 @@ ExecutionOutput Command::execute(const ExecutionInput& ei) const {
   for (const std::string& arg : args)
     trueArgs.emplace_back(resolveArg(ei, eo, arg).string());
 
+#if defined(DEBUG)
   std::cout << "OUT PATH: " << outPath << std::endl;
+  std::cout << "ERR PATH: " << errPath << std::endl;
+  std::cout << "INS PATH: " << ei.getInputStreamFile() << std::endl;
+#endif 
   // Get the runtime path and standard out file, the things used in setting up
   // the execution of the command.
   std::string runtimeStr = usesRuntime ? ei.getTestedRuntime().string() : "";
-  std::string inPathStr = usesInStr ? ei.getInputStreamFile().string() : "";
+  std::string inPathStr = fs::exists(ei.getInputStreamFile()) && usesInStr 
+                                    ? ei.getInputStreamFile().string()
+                                    : "";
   std::string outPathStr = outPath.string();
   std::string errPathStr = errPath.string();