feat: worker threads instead of child processes

components/root/root.js

@@ -31,7 +31,7 @@ export class Root extends StacheElement {
 
   get comp() {
     const loadId = this.routeData.loadId
-    console.log("route.data.loadId", loadId)
+    // console.log("route.data.loadId", loadId)
 
     if (!loadId || loadId === "root") {
       const root = document.createElement("h2")

jsdom-ssr/index-worker.js

@@ -0,0 +1,81 @@
+const { isMainThread } = require("worker_threads")
+
+const workerThread = require("./worker-thread")
+
+const { ensureDir, emptyDir, readJson } = require("fs-extra")
+const stealTools = require("steal-tools")
+const argv = require("optimist").argv
+
+main()
+
+async function main() {
+  // Create production bundle as needed
+  // Development doesn't require a build for ssr
+  if (argv.prod) {
+    await stealTools.build(
+      {},
+      {
+        bundleSteal: true,
+      },
+    )
+  }
+
+  // Create dist directory
+  await ensureDir("dist/ssr")
+
+  // Clear it
+  await emptyDir("dist/ssr")
+
+  // Read paths to generate static pages
+  // const ssgSettings = await readJson("ssg.json")
+  const ssgSettings = await readJson("ssg-max.json")
+
+  const routes = ssgSettings.routes
+
+  const initialLength = routes.length
+
+  const workerCount = 32
+
+  const workers = []
+
+  for (let i = 0; i < Math.min(routes.length, workerCount); i++) {
+    const worker = workerThread(i, !!argv.prod)
+    worker.postMessage(routes.pop())
+
+    worker.on("message", () => {
+      const route = routes.pop()
+      worker.postMessage(route)
+    })
+
+    workers.push(worker)
+  }
+
+  await Promise.all(workers.map((worker) => onExit(worker)))
+
+  console.log(`Finished: ${initialLength - routes.length}`)
+}
+
+async function onExit(worker, callback) {
+  return new Promise((resolve) => {
+    worker.once("exit", () => {
+      resolve(callback && callback())
+    })
+  })
+}
+
+// Time things:
+// npm i -D gnomon
+// npm run build | gnomon
+// pool of 32 -> 71.0153s
+// pool of 32 -> 90.5499s (while watching something on youtube)
+
+// Invalid constructor, the constructor is not part of the custom element registry
+// error comes from jsdom, file search node_modules
+// 81.6880s
+// Recreatable from 1 to 11
+/**
+ *     "http://127.0.0.1:8080/progressive-loading/cow",
+    "http://127.0.0.1:8080/progressive-loading/moo"
+ */
+
+// 400 -> 84.0856s

jsdom-ssr/index.js

@@ -1,4 +1,7 @@
+const pLimit = require("p-limit")
+
 const spawnBuildProcess = require("./spawn-build-process")
+
 const { ensureDir, emptyDir, readJson } = require("fs-extra")
 const stealTools = require("steal-tools")
 const argv = require("optimist").argv
@@ -24,11 +27,32 @@ async function main() {
   await emptyDir("dist/ssr")
 
   // Read paths to generate static pages
-  const ssgSettings = await readJson("ssg.json")
+  const ssgSettings = await readJson("ssg-max.json")
 
   const routes = ssgSettings.routes
 
-  for (const route of routes) {
-    spawnBuildProcess(route, !!argv.prod)
-  }
+  const limit = pLimit(32)
+
+  const pool = routes.map((route) =>
+    limit(function () {
+      return spawnBuildProcess(route, !!argv.prod)
+    }),
+  )
+
+  const completed = await Promise.all(pool)
+
+  console.log(`Finished: ${completed.length}`)
 }
+
+// Time things:
+// npm i -D gnomon
+// npm run build | gnomon
+
+// At 400,  155.3191s with all processes -> crash...
+// pool of 32 -> 95.6140s
+// pool of 32 -> 118.9888s (while watching something on youtube)
+
+// after
+
+// pool of 32 -> 111.2810s
+// pool of 32 -> 117.4962s <-- includes moo and cow ):

jsdom-ssr/scrape-worker.js

@@ -0,0 +1,154 @@
+const steal = require("steal")
+const setupGlobals = require("./setup-globals")
+const { outputFile, existsSync, readFileSync } = require("fs-extra")
+const getFilepath = require("../util/get-filepath")
+const argv = require("optimist").argv
+const { parentPort } = require("worker_threads")
+
+// Get url from argv
+const id = argv.id
+// Generate prod vs dev scapes
+const prod = argv.prod || false
+// Setup entry point based on prod
+const entryPoint = prod ? "production.html" : "index.html"
+
+if (id !== 0 && !id) {
+  console.error("Unexpected missing worker thread id")
+  throw new Error("Unexpected missing worker thread id")
+}
+
+let mode = ""
+let url = ""
+
+function setMode(_) {
+  mode = _
+  // console.log(`${id} - ${mode}`)
+}
+
+setMode("WAITING")
+waitForMessage()
+
+function waitForMessage() {
+  parentPort.once("message", (message) => {
+    url = message
+
+    if (!url) {
+      mode = "FINISH"
+      return
+    }
+
+    generate()
+  })
+}
+
+/**
+ * Wait for process to become idle (no async tasks are pending)
+ *
+ * This is when it is safe to scrape document
+ */
+process.on("beforeExit", (code) => {
+  if (mode === "GENERATING") {
+    scrapeDocument()
+    return
+  }
+
+  if (mode !== "WAITING" && mode !== "FINISH") {
+    console.error("Unexpected exit from worker thread")
+    throw new Error("Unexpected exit from worker thread")
+  }
+
+  while (mode === "WAITING") {}
+})
+
+async function populateDocument() {
+  // Run client-side code
+  await steal.clone().startup() // loads canjs app
+  // TODO: disable jsdom script tags?
+}
+
+// Strip steal / production bundle from entry point html file
+let captureSteal = ""
+let rootCode = ""
+let stealRegex = ""
+
+async function generate() {
+  if (!url) {
+    console.error("Unexpected missing url")
+    throw new Error("Unexpected missing url")
+  }
+
+  setMode("GENERATING")
+  console.log(url)
+
+  if (existsSync(entryPoint)) {
+    if (prod) {
+      // TODO: Create a better regex for production script
+      stealRegex = /(<script[^>]*main\.js.*?>.*?<\/script>)/i
+    } else {
+      stealRegex = /(<script[^>]*steal\/steal.*?>.*?<\/script>)/i
+    }
+
+    rootCode = readFileSync(entryPoint, { encoding: "utf8", flag: "r" }) // project"s index.html / production.html
+      .replace(stealRegex, (_, stealTag) => {
+        captureSteal = stealTag
+        return "" // remove steal script tag (re-injected before exit)
+      })
+
+    if (!/^<!doctype/i.test(rootCode)) {
+      rootCode = "<!doctype html>" + rootCode
+    }
+
+    if (rootCode.indexOf("<canjs-app") === -1) {
+      if (rootCode.indexOf("</body") !== -1) {
+        rootCode = rootCode.replace("</body", "<canjs-app></canjs-app></body")
+      } else {
+        rootCode += "<canjs-app></canjs-app>"
+      }
+    }
+  } else {
+    rootCode = `<!doctype html><title>CanJS and StealJS</title><canjs-app></canjs-app>`
+
+    if (prod) {
+      captureSteal = `<script src="/dist/bundles/can-stache-element-ssr/main.js" main></script>`
+    } else {
+      captureSteal = `<script src="/node_modules/steal/steal.js" main></script>`
+    }
+  }
+
+  // Setup JSDOM and global.window, global.document, global.location
+  setupGlobals(rootCode, url)
+
+  await populateDocument()
+}
+
+/**
+ * Once async tasks are completed, scrap document into dist
+ */
+async function scrapeDocument() {
+  setMode("SCRAPING")
+  // Write scrapped dom to dist
+  let html = window.document.documentElement.outerHTML
+
+  // Set Inert Prerendered flag
+  html = html.replace(
+    /(<head[^>]*>)/,
+    `$1<script>
+    globalThis.canStacheElementInertPrerendered = true;
+    globalThis.canMooStache = true;
+  </script>`,
+  )
+
+  // Re-inject steal before closing of body tag
+  // It's required that steal is injected at the end of body to avoid runtime errors involving `CustomElement`
+  // source: https://stackoverflow.com/questions/43836886/failed-to-construct-customelement-error-when-javascript-file-is-placed-in-head
+  html = html.replace("</body>", captureSteal + "</body>")
+
+  html = html.replace(/(<canjs-app[^>]*)>/, "$1 data-canjs-static-render>")
+  // html = html.replace("</body>", injectHydrateInZoneWithCache + "</body>")
+
+  await outputFile(`dist/ssr/${getFilepath(url, "index.html")}`, html)
+
+  parentPort.postMessage(url)
+  setMode("WAITING")
+  waitForMessage()
+}

jsdom-ssr/scrape.js

@@ -12,17 +12,17 @@ const prod = argv.prod || false
 const entryPoint = prod ? "production.html" : "index.html"
 
 // Throw if build takes too long
-const timeout = setTimeout(() => {
-  throw new Error("timed out ):")
-}, 5000).unref()
+// const timeout = setTimeout(() => {
+//   throw new Error("timed out ):")
+// }, 5000).unref()
 
 /**
  * Wait for process to become idle (no async tasks are pending)
  *
  * This is when it is safe to scrape document
  */
 process.once("beforeExit", (code) => {
-  clearTimeout(timeout)
+  // clearTimeout(timeout)
 
   // TODO: should we consider code? code === 0?
   scrapeDocument()
@@ -76,7 +76,7 @@ async function populateDocument() {
   await steal.startup() // loads canjs app
   // TODO: disable jsdom script tags?
 
-  console.log("steal - done")
+  // console.log("steal - done")
 }
 
 populateDocument()

jsdom-ssr/worker-thread.js

@@ -0,0 +1,29 @@
+const { Worker, isMainThread, parentPort } = require("worker_threads")
+const path = require("path")
+const argv = require("optimist").argv
+
+// async function onExit(worker, callback) {
+//   return new Promise((resolve) => {
+//     worker.once("exit", () => {
+//       resolve(callback && callback())
+//     })
+//   })
+// }
+
+const baseUrl = "http://127.0.0.1:8080/index.html"
+
+module.exports = function (id, prod = false) {
+  const args = ["--id", id]
+
+  if (prod) {
+    args.push("--prod", "true")
+  }
+
+  const worker = new Worker(path.join(__dirname, "scrape-worker.js"), {
+    argv: args,
+  })
+
+  return worker
+
+  // return onExit(worker)
+}

main.js

@@ -49,7 +49,7 @@ class MyRoutingApp extends StacheElement {
   }
 
   get componentToShow() {
-    console.log("route.data.page", this.routeData.page)
+    // console.log("route.data.page", this.routeData.page)
 
     // TODO: Progressive loading
     switch (this.routeData.page) {
@@ -210,7 +210,7 @@ class MyApp extends StacheElement {
     if (this.INERT_PRERENDERED) {
       return
     }
-    console.log("constructing canjs-app")
+    // console.log("constructing canjs-app")
     this.style.color = "lime"
     setTimeout(() => (this.style.color = "#552255"), 1000)
   }
@@ -219,7 +219,7 @@ class MyApp extends StacheElement {
     // non-standard lifecycle hooks won't run if this.INERT_PRERENDERED is true
     this.style.backgroundColor = "white"
     setTimeout(() => (this.style.backgroundColor = "violet"), 1000)
-    console.log("MyApp - connected")
+    // console.log("MyApp - connected")
     this.name = "canjs"
     this.appendChild(document.createElement("my-counter"))
 
@@ -233,4 +233,4 @@ ssrDefineElement("canjs-app", MyApp)
 
 ssrEnd()
 
-console.log("href", window.location.href)
+// console.log("href", window.location.href)

package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "prepare": "husky install",
     "build": "node jsdom-ssr/index.js",
+    "build-worker": "node jsdom-ssr/index-worker.js",
     "build-prod": "node jsdom-ssr/index.js --prod true",
     "build-debug": "node --inspect-brk jsdom-ssr/index.js",
     "serve": "node server.js --port 8080",