Create seperate testLoader and testCached for Node v22 compatibility with JSON assertions (#1149)

* Add test loader

* Change exec to testCached

* cleanup pathing for execution

* bump version

* lint

* fix failing test

* Fix rootEsm tests for JSON compat with 22

Author: TarikGul

CHANGELOG.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 
+## 0.82.1
+
+- Create seperate testLoader and testCached for Node v22 compatibility with JSON assertions
+
+
 ## 0.81.1
 
 - Duplicate .d.ts files into cjs

package.json

@@ -14,7 +14,7 @@
   },
   "sideEffects": false,
   "type": "module",
-  "version": "0.81.2",
+  "version": "0.82.0",
   "versions": {
     "git": "0.81.2",
     "npm": "0.81.2"

packages/dev-test/package.json

@@ -15,7 +15,7 @@
   },
   "sideEffects": false,
   "type": "module",
-  "version": "0.81.2",
+  "version": "0.82.0",
   "main": "./index.js",
   "exports": {
     "./globals.d.ts": "./src/globals.d.ts"

packages/dev-ts/package.json

@@ -15,7 +15,7 @@
   },
   "sideEffects": false,
   "type": "module",
-  "version": "0.81.2",
+  "version": "0.82.0",
   "main": "./index.js",
   "exports": {
     "./globals.d.ts": "./src/globals.d.ts"

packages/dev-ts/src/testCached.ts

@@ -0,0 +1,13 @@
+// Copyright 2017-2024 @polkadot/dev-ts authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+// Adapted from: https://nodejs.org/api/esm.html#esm_transpiler_loader
+//
+// NOTE: This assumes the loader implementation for Node.js >= 18
+
+import { loaderOptions } from './common.js';
+
+loaderOptions.isCached = true;
+
+export { resolve } from './resolver.js';
+export { load } from './testLoader.js';

packages/dev-ts/src/testLoader.ts

@@ -0,0 +1,92 @@
+// Copyright 2017-2024 @polkadot/dev-ts authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import ts from 'typescript';
+
+import { EXT_TS_REGEX, loaderOptions } from './common.js';
+
+interface Loaded {
+  format: 'commonjs' | 'module';
+  shortCircuit?: boolean;
+  source: string;
+}
+
+type NexLoad = (url: string, context: Record<string, unknown>) => Promise<Loaded>;
+
+/**
+ * Load all TypeScript files, compile via tsc on-the-fly
+ **/
+export async function load (url: string, context: Record<string, unknown>, nextLoad: NexLoad): Promise<Loaded> {
+  if (EXT_TS_REGEX.test(url)) {
+    // used the chained loaders to retrieve
+    const { source } = await nextLoad(url, {
+      ...context,
+      format: 'module'
+    });
+
+    // This ensures there is support for Node v22 while also maintaining backwards compatibility for testing.
+    const modifiedSrc = Buffer.from(source.toString().replace(/assert\s*\{\s*type:\s*'json'\s*\}/g, 'with { type: \'json\' }'), 'utf-8');
+
+    // we use a hash of the source to determine caching
+    const sourceHash = `//# sourceHash=${crypto.createHash('sha256').update(modifiedSrc as unknown as string).digest('hex')}`;
+    const compiledFile = url.includes('/src/')
+      ? fileURLToPath(
+        url
+          .replace(/\.tsx?$/, '.js')
+          .replace('/src/', '/build-loader/')
+      )
+      : null;
+
+    if (loaderOptions.isCached && compiledFile && fs.existsSync(compiledFile)) {
+      const compiled = fs.readFileSync(compiledFile, 'utf-8');
+
+      if (compiled.includes(sourceHash)) {
+        return {
+          format: 'module',
+          source: compiled
+        };
+      }
+    }
+
+    // compile via typescript
+    const { outputText } = ts.transpileModule(modifiedSrc.toString(), {
+      compilerOptions: {
+        ...(
+          url.endsWith('.tsx')
+            ? { jsx: ts.JsxEmit.ReactJSX }
+            : {}
+        ),
+        esModuleInterop: true,
+        importHelpers: true,
+        inlineSourceMap: true,
+        module: ts.ModuleKind.ESNext,
+        moduleResolution: ts.ModuleResolutionKind.NodeNext,
+        skipLibCheck: true,
+        // Aligns with packages/dev/scripts/polkadot-dev-build-ts & packages/dev/config/tsconfig
+        target: ts.ScriptTarget.ES2022
+      },
+      fileName: fileURLToPath(url)
+    });
+
+    if (loaderOptions.isCached && compiledFile) {
+      const compiledDir = path.dirname(compiledFile);
+
+      if (!fs.existsSync(compiledDir)) {
+        fs.mkdirSync(compiledDir, { recursive: true });
+      }
+
+      fs.writeFileSync(compiledFile, `${outputText}\n${sourceHash}`, 'utf-8');
+    }
+
+    return {
+      format: 'module',
+      source: outputText
+    };
+  }
+
+  return nextLoad(url, context);
+}

packages/dev/package.json

@@ -15,7 +15,7 @@
   },
   "sideEffects": false,
   "type": "module",
-  "version": "0.81.2",
+  "version": "0.82.0",
   "bin": {
     "polkadot-ci-ghact-build": "./scripts/polkadot-ci-ghact-build.mjs",
     "polkadot-ci-ghact-docs": "./scripts/polkadot-ci-ghact-docs.mjs",
@@ -50,8 +50,8 @@
   },
   "dependencies": {
     "@eslint/js": "^8.56.0",
-    "@polkadot/dev-test": "^0.81.2",
-    "@polkadot/dev-ts": "^0.81.2",
+    "@polkadot/dev-test": "^0.82.0",
+    "@polkadot/dev-ts": "^0.82.0",
     "@rollup/plugin-alias": "^5.1.0",
     "@rollup/plugin-commonjs": "^25.0.7",
     "@rollup/plugin-dynamic-import-vars": "^2.1.2",

packages/dev/scripts/polkadot-dev-run-test.mjs

@@ -157,7 +157,7 @@ try {
       : `@polkadot/dev-test/${testEnv}`
   );
 
-  execNodeTs(allFlags, nodeFlags, false, isDev ? './packages/dev-ts/build/cached.js' : undefined);
+  execNodeTs(allFlags, nodeFlags, false, isDev ? './packages/dev-ts/build/testCached.js' : '@polkadot/dev-ts/testCached');
 } catch {
   process.exit(1);
 }

packages/dev/src/rootEsm.spec.ts

@@ -3,20 +3,13 @@
 
 /// <reference types="@polkadot/dev-test/globals.d.ts" />
 
-import type * as testRoot from './root.js';
-
 import fs from 'node:fs';
 import path from 'node:path';
 
-// NOTE We don't use ts-expect-error here since the build folder may or may
-// not exist (so the error may or may not be there)
-//
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore This should only run against the compiled ouput, where this should exist
-import * as testRootBuild from '../build/root.js';
+import * as testRoot from './root.js';
 import { runTests } from './rootTests.js';
 
-runTests(testRootBuild as unknown as typeof testRoot);
+runTests(testRoot);
 
 describe('as-built output checks', (): void => {
   const buildRoot = path.join(process.cwd(), 'packages/dev/build');
@@ -94,7 +87,8 @@ describe('as-built output checks', (): void => {
             jsIdx[type].includes(
               type === 'cjs'
                 ? 'require("@polkadot/dev/rootJs/testJson.json")'
-                : "import testJson from '@polkadot/dev/rootJs/testJson.json' assert { type: 'json' };"
+                // eslint-disable-next-line no-useless-escape
+                : "import testJson from '@polkadot/dev/rootJs/testJson.json' assert { type: \'json\' };"
             )
           ).toBe(true);
         })

yarn.lock

@@ -433,7 +433,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@polkadot/dev-test@npm:^0.81.2, @polkadot/dev-test@workspace:packages/dev-test":
+"@polkadot/dev-test@npm:^0.82.0, @polkadot/dev-test@workspace:packages/dev-test":
   version: 0.0.0-use.local
   resolution: "@polkadot/dev-test@workspace:packages/dev-test"
   dependencies:
@@ -443,7 +443,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@polkadot/dev-ts@npm:^0.81.2, @polkadot/dev-ts@workspace:packages/dev-ts":
+"@polkadot/dev-ts@npm:^0.82.0, @polkadot/dev-ts@workspace:packages/dev-ts":
   version: 0.0.0-use.local
   resolution: "@polkadot/dev-ts@workspace:packages/dev-ts"
   dependencies:
@@ -458,8 +458,8 @@ __metadata:
   resolution: "@polkadot/dev@workspace:packages/dev"
   dependencies:
     "@eslint/js": "npm:^8.56.0"
-    "@polkadot/dev-test": "npm:^0.81.2"
-    "@polkadot/dev-ts": "npm:^0.81.2"
+    "@polkadot/dev-test": "npm:^0.82.0"
+    "@polkadot/dev-ts": "npm:^0.82.0"
     "@rollup/plugin-alias": "npm:^5.1.0"
     "@rollup/plugin-commonjs": "npm:^25.0.7"
     "@rollup/plugin-dynamic-import-vars": "npm:^2.1.2"