Update Check to write JSON (#12)

kaist-plrg · Feb 22, 2024 · be25d1e · be25d1e
1 parent b5e4b50
commit be25d1e
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 22 deletions.
diff --git a/src/main/scala/fhetest/Checker/ExecuteResult.scala b/src/main/scala/fhetest/Checker/ExecuteResult.scala
@@ -6,8 +6,12 @@ trait ExecuteResult
 case class Normal(res: String) extends ExecuteResult {
   override def toString: String = res
 }
-case object InterpError extends ExecuteResult
-case object PrintError extends ExecuteResult
+case object InterpError extends ExecuteResult {
+  override def toString: String = "InterpError"
+}
+case object PrintError extends ExecuteResult {
+  override def toString: String = "PrintError"
+}
 case class LibraryError(msg: String) extends ExecuteResult {
   override def toString: String = s"LibraryError: $msg"
 }

diff --git a/src/main/scala/fhetest/Checker/Utils.scala b/src/main/scala/fhetest/Checker/Utils.scala
@@ -20,3 +20,21 @@ def dumpJson[T](data: T, filename: String)(implicit
   writer: JsonWriter[T],
 ): Unit =
   dumpFile(data.toJson.prettyPrint, filename)
+
+val TEST_DIR = fhetest.TEST_DIR
+val succDir = s"$TEST_DIR/succ"
+val failDir = s"$TEST_DIR/fail"
+val psrErrDir = s"$TEST_DIR/psr_err"
+val testDirPath = Paths.get(TEST_DIR)
+val succDirPath = Paths.get(succDir)
+val failDirPath = Paths.get(failDir)
+val psrErrDirPath = Paths.get(psrErrDir)
+
+def setTestDir(): Unit = {
+  val pathLst = List(testDirPath, succDirPath, failDirPath, psrErrDirPath)
+  pathLst.foreach(path =>
+    if (!Files.exists(path)) {
+      Files.createDirectories(path)
+    },
+  )
+}
diff --git a/src/main/scala/fhetest/Command.scala b/src/main/scala/fhetest/Command.scala
@@ -172,14 +172,17 @@ case object CmdCheck extends Command("check") {
   val examples = List(
     "fhetest check tmp --SEAL --OpenFHE",
   )
+  // TODO: json option 추가
   def apply(args: List[String]): Unit = args match {
     case dir :: backendStrings => {
       val backendList = backendStrings.flatMap(parseBackend(_))
       if (backendStrings.size == backendList.size) {
         // TODO: temporary encParams. Fix after having parameter genernation.
         val encParams = EncParams(32768, 5, 65537)
-        val output = Check(dir, backendList, encParams)
-        println(output)
+        val outputs = Check(dir, backendList, encParams)
+        for output <- outputs do {
+          println(output)
+        }
       } else { println("Argument parsing error: Invalid backend.") }
     }
     case _ => println("Invalid arguments")
@@ -194,6 +197,7 @@ case object CmdTest extends Command("test") {
     "fhetest test --INT --random 10",
     "fhetest test --DOUBLE --exhaust 10",
   )
+  // TODO: json option 추가
   def apply(args: List[String]): Unit = args match {
     case Nil => println("No argument given.")
     case encTypeString :: stgString :: remain => {
@@ -209,10 +213,10 @@ case object CmdTest extends Command("test") {
       val backendList = List(Backend.SEAL, Backend.OpenFHE)
       // TODO: temporary encParams. Fix after having parameter genernation.
       val encParams = EncParams(32768, 5, 65537)
-      for program <- programs do {
-        val output = Check(program, backendList, encParams)
+      val outputs = Check(programs, backendList, encParams)
+      for (program, output) <- outputs do {
         println("=" * 80)
-        println("Program : " + program.content)
+        println("Program : " + program)
         println("-" * 80)
         println(output)
         println("=" * 80)

diff --git a/src/main/scala/fhetest/Phase/Check.scala b/src/main/scala/fhetest/Phase/Check.scala
@@ -8,6 +8,8 @@ import org.twc.terminator.SymbolTable;
 import java.nio.file.{Files, Paths};
 import java.io.{File, InputStream, ByteArrayInputStream}
 import scala.jdk.CollectionConverters._
+import spray.json._
+import spray.json.DefaultJsonProtocol._
 
 case object Check {
   def apply(
@@ -32,16 +34,16 @@ case object Check {
     programs: LazyList[T2Program],
     backends: List[Backend],
     encParams: EncParams,
-  ): LazyList[CheckResult] = {
+  ): LazyList[(String, CheckResult)] = {
+    setTestDir()
+    var i = 0
     val checkResults = for {
       program <- programs
-    } yield apply(program, backends, encParams)
-
-    // TODO: Write Json file
-    // checkResults.foreach {
-
-    // }
-
+    } yield {
+      val checkResult = getAndWriteResult(i, program, backends, encParams)
+      i = i + 1
+      (program.content, checkResult)
+    }
     checkResults
   }
 
@@ -54,22 +56,20 @@ case object Check {
     if (dir.exists() && dir.isDirectory) {
       val files = Files.list(Paths.get(directory))
       val fileList = files.iterator().asScala.toList
+      setTestDir()
+      var i = 0
       val checkResults = for {
         filePath <- fileList.to(LazyList)
       } yield {
         val fileStr = Files.readAllLines(filePath).asScala.mkString("")
-        val checkResult = apply(T2Program(fileStr), backends, encParams)
+        val program = T2Program(fileStr)
+        val checkResult = getAndWriteResult(i, program, backends, encParams)
+        i = i + 1
         val pgmStr = "-" * 10 + " Program " + "-" * 10 + "\n" + fileStr + "\n"
         val reportStr = checkResult.toString + "\n"
         pgmStr + reportStr
 
       }
-
-      // TODO: Write Json file
-      // checkResults.foreach {
-
-      // }
-
       checkResults
     } else {
       throw new Exception("Directory does not exist or is not a directory")
@@ -139,4 +139,61 @@ case object Check {
         }
       },
     )
+
+  def getAndWriteResult(
+    i: Int,
+    program: T2Program,
+    backends: List[Backend],
+    encParams: EncParams,
+  ): CheckResult = {
+    val pgm_info = Map(
+      ("programId" -> i.toString),
+      ("program" -> program.content),
+    )
+    val checkResult = apply(program, backends, encParams)
+    checkResult match {
+      case Same(res) => {
+        val (expectedLst, obtainedLst) = res.partition(_.backend == "CLEAR")
+        val expected_res = expectedLst.apply(0).result
+        val result = pgm_info ++ Map(
+          "result" -> "Success",
+          "failedLibraires" -> "0",
+          "failures" -> JsArray().prettyPrint,
+          "expected" -> expected_res.toString,
+        )
+        val succFilename = s"$succDir/$i.json"
+        dumpJson(result, succFilename)
+      }
+      case Diff(res) => {
+        val (expectedLst, obtainedLst) = res.partition(_.backend == "CLEAR")
+        val expected = expectedLst.apply(0)
+        val diffResults = obtainedLst.filter(isDiff(expected, _))
+        val failures = diffResults.map(r =>
+          Map(
+            ("library" -> r.backend),
+            ("failedResult" -> r.result.toString),
+          ),
+        )
+        val result = pgm_info ++ Map(
+          "result" -> "Fail",
+          "failedLibraires" -> diffResults.size.toString,
+          "failures" -> failures.toJson.prettyPrint,
+          "expected" -> expected._2.toString,
+        )
+        val failFilename = s"$failDir/$i.json"
+        dumpJson(result, failFilename)
+      }
+      case ParserError(_) => {
+        val result = pgm_info ++ Map(
+          "result" -> "ParseError",
+          "failedLibraires" -> "NaN",
+          "failures" -> JsArray().prettyPrint,
+          "expected" -> "",
+        )
+        val psrErrFilename = s"$psrErrDir/$i.json"
+        dumpJson(result, psrErrFilename)
+      }
+    }
+    checkResult
+  }
 }
diff --git a/src/main/scala/fhetest/package.scala b/src/main/scala/fhetest/package.scala
@@ -5,6 +5,7 @@ val BASE_DIR = settings.ProjectSettings.projectHome
 
 /** log directory */
 val LOG_DIR = s"$BASE_DIR/logs"
+val TEST_DIR = s"$LOG_DIR/test"
 
 /** T2 directory */
 val T2_DIR = s"$BASE_DIR/src/main/java/T2-FHE-Compiler-and-Benchmarks"