Virtualize path and file system operations

bin/test-compile.mjs

@@ -0,0 +1,92 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { inspect } from "node:util";
+
+// Update the relative path here if this file is moved.
+const projectPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+
+async function loadStandardLibrary() {
+  const compilePath = path.join(projectPath, "hkmc2/shared/src/test/mlscript-compile");
+  const files = await Promise.all(
+    (await fs.readdir(compilePath))
+      .filter((fileName) => {
+        const ext = path.extname(fileName);
+        return ext === ".mls" || ext === ".mjs";
+      })
+      .map(async (fileName) => ({
+        path: `/std/${fileName}`,
+        content: await fs.readFile(path.join(compilePath, fileName), "utf-8"),
+      }))
+  );
+  const preludePath = path.join(projectPath, "hkmc2/shared/src/test/mlscript/decls/Prelude.mls");
+  files.push({ path: "/std/Prelude.mls", content: await fs.readFile(preludePath, "utf-8") });
+  return files;
+}
+
+async function importMLscript() {
+  // Check if "./hkmc2/js/target/scala-3.7.3/hkmc2-opt/MLscript.mjs" exists.
+  // If not, check "./hkmc2/js/target/scala-3.7.3/hkmc2-fastopt/MLscript.mjs"
+  const mlscriptPath = path.join(projectPath, "hkmc2/js/target/scala-3.7.3/hkmc2-opt/MLscript.mjs");
+  const mlscriptFastOptPath = path.join(projectPath, "hkmc2/js/target/scala-3.7.3/hkmc2-fastopt/MLscript.mjs");
+  try {
+    await fs.access(mlscriptPath);
+    return await import(mlscriptPath);
+  } catch {
+    try {
+      await fs.access(mlscriptFastOptPath);
+      return await import(mlscriptFastOptPath);
+    } catch {
+      throw new Error(
+        `MLscript module not found. Please build the project first.`
+      );
+    }
+  }
+}
+
+async function main() {
+  const { Compiler, InMemoryFileSystem, Paths } = await importMLscript();
+  const fileSystem = InMemoryFileSystem(
+    (await loadStandardLibrary()).map(({ path, content }) => [path, content])
+  );
+  const paths = new Paths(
+    "/std/Prelude.mls",
+    "/std/Runtime.mjs",
+    "/std/Term.mjs"
+  );
+  const compiler = new Compiler(fileSystem, paths);
+  const program = `import "./std/Option.mls"
+import "./std/Stack.mls"
+import "./std/Predef.mls"
+
+open Stack
+open Option
+open Predef
+
+fun findFirst(xs, f) = if xs is
+  Nil then None
+  Cons(x, xs') and
+    f(x) then Some(x)
+    else findFirst(xs', f)
+
+let nums = 1 :: 2 :: 3 :: 4 :: 5 :: Nil
+let result = nums \\findFirst of x => x * 6 is 24
+`;
+  console.log(bold(red("Source Program:")));
+  console.log(program);
+  fileSystem.write("/test.mls", program);
+  console.log(bold(red("Dianostics:")));
+  console.log(inspect(compiler.compile("/test.mls"), { depth: null }));
+  console.log(bold(red("Compiled JavaScript:")));
+  console.log(fileSystem.read("/test.mjs"));
+}
+
+main();
+
+function red(text) {
+  return `\x1b[31m${text}\x1b[0m`;
+}
+
+function bold(text) {
+  return `\x1b[1m${text}\x1b[0m`;
+}

build.sbt

@@ -1,4 +1,5 @@
 import Wart._
+import org.scalajs.linker.interface.OutputPatterns
 
 enablePlugins(ScalaJSPlugin)
 
@@ -42,7 +43,8 @@ lazy val hkmc2 = crossProject(JSPlatform, JVMPlatform).in(file("hkmc2"))
 
     libraryDependencies += "io.methvin" % "directory-watcher" % directoryWatcherVersion,
     libraryDependencies += "io.methvin" %% "directory-watcher-better-files" % directoryWatcherVersion,
-    libraryDependencies += "com.lihaoyi" %%% "fansi" % "0.4.0",
+    libraryDependencies += "com.lihaoyi" %%% "fansi" % "0.5.0", // Scala.js or Scala-Native
+    libraryDependencies += "com.lihaoyi" %%% "sourcecode" % "0.4.2", // Scala.js / Scala Native
     libraryDependencies += "com.lihaoyi" %% "os-lib" % "0.9.3",
 
     libraryDependencies += "org.scalactic" %%% "scalactic" % scalaTestVersion,
@@ -57,6 +59,13 @@ lazy val hkmc2 = crossProject(JSPlatform, JVMPlatform).in(file("hkmc2"))
   )
   .jvmSettings(
   )
+  .jsSettings(
+    scalaJSLinkerConfig ~= {
+      _.withModuleKind(ModuleKind.ESModule)
+       .withOutputPatterns(OutputPatterns.fromJSFile("MLscript.mjs"))
+    },
+    libraryDependencies += "org.scala-js" %%% "scalajs-dom" % "2.2.0",
+  )
   .dependsOn(core)
 
 lazy val hkmc2JVM = hkmc2.jvm

core/shared/main/scala/utils/package.scala

@@ -206,10 +206,12 @@ package object utils {
     }
   }
 
+  // * The goal of these is to avoid the use of varargs, which I've found to be a source of
+  // * overhead in the past, due to the allocation of intermediate arrays.
+  // * Remains to be seen if using these is always (or ever?) necessarily a win.
   implicit class MutSetObjectHelpers(self: mutable.Set.type) {
     def single[A](a: A): mutable.Set[A] = mutable.Set.empty[A] += a
   }
-
   implicit class SetObjectHelpers(self: Set.type) {
     def single[A](a: A): Set[A] = (Set.newBuilder[A] += a).result()
   }
@@ -222,6 +224,11 @@ package object utils {
   implicit class SortedMapObjectHelpers(self: SortedMap.type) {
     def single[A: Ordering, B](ab: A -> B): SortedMap[A, B] = (SortedMap.newBuilder[A, B] += ab).result()
   }
+  implicit class VectorObjectHelpers(self: Vector.type) {
+    def single[A](a: A): Vector[A] = a +: Vector.empty
+    def double[A](a: A, b: A): Vector[A] = a +: b +: Vector.empty
+    def triple[A](a: A, b: A, c: A): Vector[A] = a +: b +: c +: Vector.empty
+  }
 
   def TODO(msg: Any): Nothing = throw new NotImplementedError(
     msg.toString + s" (of class ${msg.getClass().getSimpleName()})")

hkmc2/js/src/main/scala/hkmc2/Compiler.scala

@@ -0,0 +1,72 @@
+package hkmc2
+
+import scala.util.Try
+import scala.scalajs.js.annotation.*
+import org.scalajs.dom
+import org.scalajs.dom.document
+import mlscript.utils._
+import mlscript.utils.shorthands._
+import scala.util.matching.Regex
+import scala.scalajs.js, js.JSConverters.*
+import scala.collection.immutable
+import scala.collection.mutable.Map as MutMap
+
+import io.*
+import scala.collection.mutable.{ArrayBuffer, Buffer}
+
+@JSExportTopLevel("Compiler")
+class Compiler(fs: FileSystem, paths: MLsCompiler.Paths):
+  private given Config = Config.default
+
+  private given FileSystem = fs
+
+  private var pathDiagnosticsMap = MutMap.empty[Str, (Int, Buffer[Diagnostic])]
+
+  private def mkRaise(path: io.Path): Raise = d =>
+    pathDiagnosticsMap.getOrElseUpdate(path.toString, (pathDiagnosticsMap.size, Buffer.empty))._2 += d
+
+  private val compiler = MLsCompiler(paths, mkRaise)
+
+  private def collectDiagnostics(): js.Array[js.Dynamic] =
+    pathDiagnosticsMap.toArray.sortBy(_._2._1).map:
+      case (path, (_, diagnostics)) => js.Dynamic.literal(
+        path = path,
+        diagnostics = diagnostics.iterator.map: d =>
+          js.Dynamic.literal(
+            kind = d.kind.toString().toLowerCase(),
+            source = d.source.toString().toLowerCase(),
+            mainMessage = d.theMsg,
+            allMessages = d.allMsgs.iterator.map:
+              case (message, loc) =>
+                lazy val ctx = ShowCtx.mk:
+                  message.bits.collect:
+                    case Message.Code(t) => t
+                js.Dynamic.literal(
+                  messageBits = message.bits.map:
+                    case Message.Text(text) => js.Dynamic.literal(text = text)
+                    case Message.Code(ty) => ty.showIn(0)(using ctx)
+                  .toJSArray,
+                  location = loc match
+                    case S(loc) => js.Dynamic.literal(
+                      start = loc.spanStart,
+                      end = loc.spanEnd
+                    )
+                    case N => null
+                )
+            .toJSArray
+          )
+        .toJSArray)
+    .toJSArray
+
+  @JSExport
+  def compile(filePath: Str): js.Array[js.Dynamic] =
+    compiler.compileModule(Path(filePath))
+    val perFileDiagnostics = collectDiagnostics()
+    pathDiagnosticsMap = MutMap.empty
+    perFileDiagnostics
+
+@JSExportTopLevel("Paths")
+final class Paths(prelude: Str, runtime: Str, term: Str) extends MLsCompiler.Paths:
+  val preludeFile = Path(prelude)
+  val runtimeFile = Path(runtime)
+  val termFile = Path(term)

hkmc2/js/src/main/scala/hkmc2/io/InMemoryFileSystem.scala

@@ -0,0 +1,41 @@
+package hkmc2.io
+
+import mlscript.utils._, shorthands._
+import collection.mutable.Map as MutMap
+import scala.scalajs.js, js.annotation.JSExport, js.JSConverters.*
+import scala.scalajs.js.annotation.JSExportTopLevel
+
+/**
+ * In-memory file system for testing and web compiler. Stores files as a map
+ * from path strings to content strings. Note that separators are not normalized.
+ */
+class InMemoryFileSystem(initialFiles: Map[String, String]) extends FileSystem:
+  // We assume that all paths are normalized here.
+  private val files: MutMap[String, String] = MutMap.from(initialFiles)
+
+  def read(path: Path): String = read(path.toString)
+
+  def write(path: Path, content: String): Unit =
+    write(path.toString, content)
+
+  def exists(path: Path): Bool = files.contains(path.toString)
+
+  @JSExport("write")
+  def write(path: Str, content: Str): Unit =
+    files(path) = content
+
+  @JSExport("read")
+  def read(path: Str): Str =
+    files.getOrElse(path, throw new FileSystem.FileNotFoundException(Path(path)))
+
+  @JSExport("list")
+  def list: js.Array[Str] = allFiles.keys.toJSArray
+
+  /** Get all files (for debugging) */
+  def allFiles: Map[String, String] = files.toMap
+
+object InMemoryFileSystem:
+  /** Create an empty in-memory file system. */
+  @JSExportTopLevel("InMemoryFileSystem")
+  def apply(files: js.Array[js.Tuple2[Str, Str]]): InMemoryFileSystem =
+    new InMemoryFileSystem(files.map(t => t._1 -> t._2).toMap)

hkmc2/js/src/main/scala/hkmc2/io/PlatformPath.scala

@@ -0,0 +1,10 @@
+package hkmc2.io
+
+/**
+ * Platform-specific factory for creating Path instances
+ */
+private[io] object PathFactory:
+  def fromString(str: String) = new VirtualPath(str)
+  def separator: String = VirtualPath.sep
+  def relPathFromString(str: String) = new VirtualRelPath(str)
+  def relPathUp = new VirtualRelPath("..")

hkmc2/js/src/main/scala/hkmc2/io/VirtualPath.scala

@@ -0,0 +1,113 @@
+package hkmc2.io
+
+import scala.scalajs.js
+import mlscript.utils._, shorthands._
+import VirtualPath.sep
+
+/**
+ * Pure JavaScript implementation of Path without using Node.js path module
+ */
+private[io] class VirtualPath(val pathString: String) extends Path:
+  private def normalizePath(path: String): String =
+    if path.isEmpty then path
+    else
+      // Split by separator and filter out empty segments
+      val segments = path.split(sep).filter(_.nonEmpty)
+      val isAbs = path.startsWith(sep)
+
+      // Resolve . and .. segments
+      val normalized = segments.foldLeft(List.empty[String]): (acc, seg) =>
+        seg match
+          case "." => acc  // Current directory, skip it
+          case ".." =>
+            // Parent directory, pop the last segment if possible
+            if acc.isEmpty || acc.last == ".." then acc :+ seg
+            else acc.dropRight(1)
+          case _ => acc :+ seg
+
+      if isAbs then sep + normalized.mkString(sep)
+      else if normalized.isEmpty then "."
+      else normalized.mkString(sep)
+
+  override def toString: String = pathString
+
+  def last: String =
+    val idx = pathString.lastIndexOf(sep)
+    if idx < 0 then pathString
+    else pathString.substring(idx + 1)
+
+  def baseName: String =
+    val filename = last
+    val dotIdx = filename.lastIndexOf('.')
+    if dotIdx <= 0 then filename  // .hidden files or no extension
+    else filename.substring(0, dotIdx)
+
+  def ext: String =
+    val filename = last
+    val dotIdx = filename.lastIndexOf('.')
+    if dotIdx <= 0 then ""  // .hidden files or no extension
+    else filename.substring(dotIdx + 1)
+
+  def up: Path =
+    val idx = pathString.lastIndexOf(sep)
+    if idx < 0 then new VirtualPath(".")
+    else if idx == 0 then new VirtualPath(sep)  // root case
+    else new VirtualPath(pathString.substring(0, idx))
+
+  def /(relPath: RelPath): Path =
+    val combined = if pathString.endsWith(sep) then
+      pathString + relPath.toString
+    else
+      pathString + sep + relPath.toString
+    new VirtualPath(normalizePath(combined))
+
+  def /(fragment: String): Path =
+    val combined = if pathString.endsWith(sep) then
+      pathString + fragment
+    else
+      pathString + sep + fragment
+    new VirtualPath(normalizePath(combined))
+
+  def relativeTo(base: Path): Opt[RelPath] =
+    try
+      val baseSegs = base.segments
+      val targetSegs = segments
+
+      // Find common prefix
+      var i = 0
+      while i < baseSegs.length && i < targetSegs.length && baseSegs(i) == targetSegs(i) do
+        i += 1
+
+      // Build relative path
+      val upCount = baseSegs.length - i
+      val ups = List.fill(upCount)("..")
+      val downs = targetSegs.drop(i)
+
+      val relSegs = ups ++ downs
+      if relSegs.isEmpty then S(new VirtualRelPath("."))
+      else S(new VirtualRelPath(relSegs.mkString(sep)))
+    catch case _: Exception => N
+
+  def segments: Ls[String] =
+    pathString.split(sep).toList.filter(_.nonEmpty)
+
+  def isAbsolute: Bool = pathString.startsWith(sep)
+
+private[io] object VirtualPath:
+  val sep = "/"
+
+/**
+ * Pure JavaScript implementation of RelPath without using Node.js path module
+ */
+private[io] class VirtualRelPath(val pathString: String) extends RelPath:
+  override def toString: String = pathString
+
+  def segments: Ls[String] =
+    pathString.split(sep).toList.filter(_.nonEmpty)
+
+  def /(other: RelPath): RelPath =
+    val combined = if pathString.endsWith(sep) then
+      pathString + other.toString
+    else
+      pathString + sep + other.toString
+    new VirtualRelPath(combined)