App E2E tests, build restructure, fix LangoustineApp on Native (#65)

build.sbt

@@ -40,7 +40,7 @@ val V = new {
   val http4s          = "0.23.16"
   val laminar         = "0.14.2"
   val decline         = "2.3.0"
-  val jsoniter        = "2.17.3"
+  val jsoniter        = "2.17.4"
   val weaver          = "0.8.0"
   val http4sJdkClient = "0.7.0"
   val organizeImports = "0.6.0"
@@ -78,7 +78,8 @@ lazy val root = project
   .aggregate(app.projectRefs*)
   .aggregate(tracer.projectRefs*)
   .aggregate(tracerShared.projectRefs*)
-  .aggregate(tracerTests.projectRefs*)
+  .aggregate(tracerFrontend.projectRefs*)
+  .aggregate(tests.projectRefs*)
   .settings(noPublishing)
 
 lazy val docs = project
@@ -154,6 +155,54 @@ lazy val app = projectMatrix
   .jsPlatform(V.scalaVersions)
   .nativePlatform(V.scalaVersions)
 
+lazy val tests = projectMatrix
+  .in(file("modules/tests"))
+  .dependsOn(app)
+  .defaultAxes(V.default*)
+  .settings(enableSnapshots)
+  .jvmPlatform(
+    V.jvmScalaVersions,
+    Seq.empty,
+    _.dependsOn(tracer.jvm(V.dynScalaVersion))
+  )
+  .jsPlatform(V.scalaVersions)
+  .nativePlatform(V.scalaVersions)
+  .settings(noPublishing)
+  .settings(
+    libraryDependencies += "org.http4s" %% "http4s-jdk-http-client" % V.http4sJdkClient % Test,
+    libraryDependencies += "com.disneystreaming" %%% "weaver-cats" % V.weaver % Test,
+    testFrameworks += new TestFramework("weaver.framework.CatsEffect"),
+    Test / fork := virtualAxes.value.contains(VirtualAxis.jvm),
+    Test / envVars := Map(
+      "EXAMPLE_NATIVE" -> (example.native(
+        V.dynScalaVersion
+      ) / Compile / nativeLink).value.toString,
+      "EXAMPLE_JVM" -> (example.jvm(
+        V.dynScalaVersion
+      ) / Compile / assembly).value.toString,
+      "EXAMPLE_JS" -> (example.js(
+        V.dynScalaVersion
+      ) / Compile / fastOptJS).value.data.toString
+    )
+  )
+
+lazy val example = projectMatrix
+  .in(file("modules/example"))
+  .dependsOn(app)
+  .defaultAxes(V.default*)
+  .settings(enableSnapshots)
+  .jvmPlatform(V.jvmScalaVersions)
+  .jsPlatform(
+    V.scalaVersions,
+    Seq(
+      scalaJSUseMainModuleInitializer := true,
+      scalaJSLinkerConfig ~= (_.withModuleKind(ModuleKind.CommonJSModule))
+    )
+  )
+  .nativePlatform(V.scalaVersions)
+  .settings(noPublishing)
+  .settings(version := "dev")
+
 lazy val generate = projectMatrix
   .in(file("modules/generate"))
   .dependsOn(meta)
@@ -231,6 +280,7 @@ lazy val tracerFrontend = projectMatrix
   .dependsOn(tracerShared)
   .defaultAxes(V.default*)
   .settings(
+    Compile / doc / sources             := Seq.empty,
     name                                := "langoustine-tracer-frontend",
     libraryDependencies += "com.raquo" %%% "laminar" % V.laminar,
     scalaJSUseMainModuleInitializer     := true
@@ -252,20 +302,6 @@ lazy val tracerShared = projectMatrix
   .jsPlatform(V.scalaVersions)
   .jvmPlatform(V.jvmScalaVersions)
 
-lazy val tracerTests = projectMatrix
-  .in(file("modules/tracer/tests"))
-  .defaultAxes(V.default*)
-  .settings(enableSnapshots)
-  .dependsOn(tracer)
-  .settings(
-    libraryDependencies += "org.http4s" %%% "http4s-ember-client" % V.http4s % Test,
-    libraryDependencies += "com.disneystreaming" %% "weaver-cats" % V.weaver % Test,
-    libraryDependencies += "org.http4s" %% "http4s-jdk-http-client" % V.http4sJdkClient % Test,
-    testFrameworks += new TestFramework("weaver.framework.CatsEffect")
-  )
-  .jvmPlatform(V.jvmScalaVersions)
-  .settings(noPublishing)
-
 val scalafixRules = Seq(
   "OrganizeImports",
   "DisableSyntax",
@@ -278,7 +314,9 @@ val CICommands = Seq(
   "scalafmtCheckAll",
   "clean",
   "compile",
-  "test"
+  "tests/test",
+  "testsJS/test",
+  "testsNative/test"
 ).mkString(";")
 
 val PrepareCICommands = Seq(
@@ -294,6 +332,9 @@ addCommandAlias(
 addCommandAlias("ci", CICommands)
 addCommandAlias("buildWebsite", "docs/unidoc")
 addCommandAlias("preCI", PrepareCICommands)
+addCommandAlias("testTracer", "tests/testOnly tests.tracer.*")
+addCommandAlias("testCore", "tests/testOnly tests.core.*")
+addCommandAlias("testE2E", "tests/testOnly tests.e2e.*")
 
 import sbtwelcome.*
 
@@ -315,11 +356,17 @@ logo :=
     |""".stripMargin
 
 usefulTasks := Seq(
-  UsefulTask("a", "generateLSP", "Regenerate LSP definitions"),
-  UsefulTask("a", "buildWebsite", "Build website"),
-  UsefulTask("b", "preCI", "Reformat and apply Scalafix rules"),
+  UsefulTask("gl", "generateLSP", "Regenerate LSP definitions"),
+  UsefulTask("bw", "buildWebsite", "Build website"),
+  UsefulTask("tt", "testTracer", "Run Tracer's backend tests"),
+  UsefulTask(
+    "te",
+    "testE2E",
+    "Run LangoustineApp E2E tests that launch a separate process"
+  ),
+  UsefulTask("fx", "preCI", "Reformat and apply Scalafix rules"),
   UsefulTask(
-    "c",
+    "p",
     "publishLocal",
     "Publish all modules locally"
   )

docs/_docs/getting_started.md

@@ -188,15 +188,15 @@ a specified command, and connecting to it via STDIN/STDOUT.
 
 As a first iteration, we can just use Scala CLI itself!
 
-```
+```text
 $ scala-cli run lsp.definition.scala
 ```
 
 You could give this command to the editor of your choosing and it will launch the server for you.
 
 Note that with Scala CLI you can package it even easier into a bootstrap jar:
 
-```
+```text
 $ scli package . -f -o LSP
 ```
 
@@ -208,7 +208,7 @@ Another way is to publish our LSP server as a JVM app and use [Coursier](https:/
 
 When we publish our application to Maven Central, it can be launched as easily as 
 
-```
+```text
 $ cs launch com.example::my-lsp:latest.release
 ```
 
@@ -234,7 +234,7 @@ a JavaScript/TypeScript library for example.
 
 To package our LSP into a single uber-JS file, we can run this command:
 
-```
+```text
 $ scli package . --js --js-module-kind common -f -o LSP.js
 ```
 
@@ -268,6 +268,30 @@ correct distribution from Node.js' website.
 
 This is a very interesting distribution mechanism and that's how an LSP for Tree Sitter grammars is distributed: https://github.com/keynmol/grammar-js-lsp/releases/tag/v0.0.3
 
+### Packaging as a native application
+
+As of quite recently, foundational libraries that power the transport mechanism in langoustine have been
+published for Scala Native.
+
+With that, we can easily package our example as a truly native app, without any dependencies:
+
+```text
+$ scli package lsp.definition.scala --native -f -o LSP.native
+```
+
+We can verify that it is indeed a binary and it has no external dependencies:
+
+```text
+$ file LSP.native
+LSP.native: Mach-O 64-bit executable arm64 # on your machine it will be different
+
+$ l --no-user LSP.native
+.rwxr-xr-x 21M 17 Sep 11:58  LSP.native
+```
+
+Both Scala Native and the Langoustine support for it are experimental, so do give it a go and 
+report any issues!
+
 ## Editor integration
 
 This section eagerly awaits your contributions!

modules/app/src/main/scala/DispatcherCommunicate.scala

@@ -19,4 +19,6 @@ private[app] class DispatcherCommunicate(
   ): Future[Unit] = disp.unsafeToFuture(target.notification(notif, in))
   override def request[X <: LSPRequest](req: X, in: req.In): Future[req.Out] =
     disp.unsafeToFuture(target.request(req, in))
+
+  override def shutdown: Future[Unit] = disp.unsafeToFuture(target.shutdown)
 end DispatcherCommunicate

modules/app/src/main/scala/LangoustineApp.scala

@@ -33,16 +33,24 @@ trait LangoustineApp extends IOApp with LangoustineApp.Config:
 end LangoustineApp
 
 object LangoustineApp:
+  opaque type Shutdown = IO[Unit]
+  object Shutdown:
+    extension (s: Shutdown) def initiate: IO[Unit] = s
+
   trait Config extends LangoustineAppPlatform:
     def lspBufferSize: Int                           = 2048
     def out: FS2.Pipe[cats.effect.IO, Byte, Nothing] = FS2.io.stdout[IO]
 
   trait FromFuture extends IOApp with Config:
     def server(args: List[String]): Future[LSPBuilder[Future]]
 
-    private def bindFutureServer(builder: LSPBuilder[Future], to: Channel[IO]) =
+    private def bindFutureServer(
+        builder: LSPBuilder[Future],
+        to: Channel[IO],
+        shutdown: IO[Unit]
+    ) =
       Dispatcher[IO].evalMap { disp =>
-        val comms       = Communicate.channel(to)
+        val comms       = Communicate.channel(to, shutdown)
         val futureComms = DispatcherCommunicate(disp, comms)
         val endpoints   = builder.build(futureComms)
 
@@ -63,12 +71,12 @@ object LangoustineApp:
       }.void
 
     override def run(args: List[String]): IO[ExitCode] =
-      FS2.Stream
+      fs2.Stream
         .resource(
           Resource
             .eval(IO.fromFuture(IO(server(args))))
-            .map { builder => (channel: Channel[IO]) =>
-              bindFutureServer(builder, channel)
+            .map { builder => (channel: Channel[IO], shutdown: IO[Unit]) =>
+              bindFutureServer(builder, channel, shutdown)
             }
         )
         .flatMap { chFun =>
@@ -92,30 +100,37 @@ object LangoustineApp:
 
   private[app] def create(
       bufferSize: Int,
-      builder: LSPBuilder[IO] | (Channel[IO] => Resource[IO, Unit]),
+      builder: LSPBuilder[IO] | ((Channel[IO], IO[Unit]) => Resource[IO, Unit]),
       in: FS2.Stream[IO, Byte],
       out: FS2.Pipe[IO, Byte, Nothing]
-  ): FS2.Stream[cats.effect.IO, Nothing] =
-    FS2Channel[IO](bufferSize, None)
-      .flatMap { channel =>
-        builder match
-          case l: LSPBuilder[IO] =>
-            FS2.Stream.resource(Resource.eval(l.bind(channel)))
-          case other: (Channel[IO] => Resource[IO, Unit]) =>
-            FS2.Stream.resource(other(channel)).as(channel)
-      }
-      .flatMap(channel =>
-        FS2.Stream
-          .eval(IO.never) // running the server forever
-          .concurrently(
-            in
-              .through(lsp.decodePayloads)
-              .through(channel.input)
-          )
-          .concurrently(
-            channel.output
-              .through(lsp.encodePayloads)
-              .through(out)
+  ): FS2.Stream[cats.effect.IO, Unit] =
+    fs2.Stream
+      .eval(IO.deferred[Boolean])
+      .flatMap { latch =>
+        FS2Channel[IO](bufferSize, None)
+          .flatMap { channel =>
+            builder match
+              case l: LSPBuilder[IO] =>
+                FS2.Stream.resource(
+                  Resource.eval(l.bind(channel, latch.complete(true).void))
+                )
+              case other: ((Channel[IO], IO[Unit]) => Resource[IO, Unit]) =>
+                FS2.Stream
+                  .resource(other(channel, latch.complete(true).void))
+                  .as(channel)
+          }
+          .flatMap(channel =>
+            fs2.Stream
+              .eval(latch.get)
+              .concurrently(
+                in.through(lsp.decodePayloads).through(channel.input)
+              )
+              .concurrently(
+                channel.output
+                  .through(lsp.encodePayloads)
+                  .through(out)
+              )
           )
-      )
+      }
+      .void
 end LangoustineApp

modules/app/src/main/scalajs/LangoustineAppPlatform.scala

@@ -1,9 +1,11 @@
 package langoustine.lsp.app
 
 import _root_.fs2 as FS2
+import cats.effect.IO
 
 private[app] trait LangoustineAppPlatform:
   self: LangoustineApp.Config =>
 
-  def in: FS2.Stream[cats.effect.IO, Byte] =
-    FS2.io.stdin[cats.effect.IO]
+  def in: fs2.Stream[IO, Byte] =
+    FS2.io.stdin[IO]
+end LangoustineAppPlatform

modules/app/src/main/scalajvm/LangoustineAppPlatform.scala

@@ -8,5 +8,6 @@ private[app] trait LangoustineAppPlatform:
 
   def inBufferSize: Int = 512
 
-  def in: FS2.Stream[cats.effect.IO, Byte] =
+  def in: fs2.Stream[IO, Byte] =
     FS2.io.stdin[IO](inBufferSize)
+end LangoustineAppPlatform

modules/app/src/main/scalanative/LangoustineAppPlatform.scala

@@ -5,7 +5,11 @@ import scala.concurrent.duration.*
 import fs2.{Chunk, Stream}
 
 import scalanative.posix
+import scalanative.libc
+import scalanative.unsigned.*
+import scala.scalanative.runtime.ByteArray
 import fs2.Pull
+import langoustine.lsp.app.LangoustineApp.Shutdown
 
 private[app] trait LangoustineAppPlatform:
   self: LangoustineApp.Config =>

modules/example/src/main/scala/Example.scala

@@ -0,0 +1,37 @@
+import langoustine.lsp.*
+import langoustine.lsp.all.*
+import langoustine.lsp.app.*
+import jsonrpclib.fs2.*
+
+import cats.effect.*
+
+object MyServer extends LangoustineApp.Simple:
+  override def server =
+    IO.ref(Set.empty[String]).map(myLSP)
+
+def myLSP(files: Ref[IO, Set[String]]) =
+  LSPBuilder
+    .create[IO]
+    .handleRequest(initialize) { in =>
+      sendMessage(in.toClient, "ready to initialise!") *>
+        IO {
+          InitializeResult(
+            capabilities = ServerCapabilities(textDocumentSync =
+              Opt(TextDocumentSyncKind.Full)
+            ),
+            serverInfo = Opt(InitializeResult.ServerInfo("My first LSP!"))
+          )
+        }
+    }
+    .handleNotification(textDocument.didOpen) { in =>
+      val documentUri = in.params.textDocument.uri.value
+      files.updateAndGet(_ + documentUri).map(_.size).flatMap { count =>
+        sendMessage(in.toClient, s"In total, $count files registered!")
+      }
+    }
+
+def sendMessage(back: Communicate[IO], msg: String) =
+  back.notification(
+    window.showMessage,
+    ShowMessageParams(MessageType.Info, msg)
+  )