Add groundwork for source map support

.changeset/mighty-ligers-beam.md

@@ -0,0 +1,5 @@
+---
+'wmr': patch
+---
+
+Rewrite internal source map handling. This adds full support for source maps during `development` and `production` and ensures that `.map` files are served correctly.

package.json

@@ -16,7 +16,8 @@
 		"demo": "yarn workspace @examples/demo run",
 		"docs": "yarn workspace docs",
 		"iso": "yarn workspace preact-iso",
-		"ci": "yarn wmr build && yarn --check-files && yarn demo build:prod"
+		"ci": "yarn wmr build && yarn --check-files && yarn demo build:prod",
+		"postinstall": "patch-package --exclude 'nothing'"
 	},
 	"eslintConfig": {
 		"extends": [
@@ -91,6 +92,7 @@
 		"eslint-plugin-prettier": "^3.1.4",
 		"husky": "^4.2.5",
 		"lint-staged": "^10.2.11",
+		"patch-package": "^6.4.7",
 		"prettier": "^2.0.5"
 	}
 }

packages/wmr/package.json

@@ -82,6 +82,7 @@
 		"semver": "^7.3.2",
 		"simple-code-frame": "^1.1.1",
 		"sirv": "^1.0.6",
+		"sourcemap-codec": "^1.4.8",
 		"stylis": "^4.0.10",
 		"sucrase": "^3.17.0",
 		"tar-stream": "^2.1.3",

packages/wmr/src/lib/acorn-traverse.js

@@ -3,6 +3,7 @@ import * as jsxWalk from 'acorn-jsx-walk';
 import MagicString from 'magic-string';
 import * as astringLib from 'astring';
 import { codeFrame } from './output-utils.js';
+import { posix } from 'path';
 
 /**
  * @fileoverview
@@ -100,7 +101,16 @@ let codeGenerator = {
 	// import(source)
 	ImportExpression(node, state) {
 		state.write('import(');
-		this[node.source.type](node.source, state);
+
+		// TODO: Sometimes this seems to have a source and sometimes
+		// an expression. I don't understand why. The expression seems
+		// to be only set when calling `t.importExpression()`
+		if (node.source) {
+			this[node.source.type](node.source, state);
+		} else {
+			this[node.expression.type](node.expression, state);
+		}
+
 		state.write(')');
 	},
 	JSXFragment(node, state) {
@@ -745,8 +755,10 @@ export function transform(
 	function getSourceMap() {
 		if (!map) {
 			map = out.generateMap({
-				includeContent: false,
-				source: sourceFileName
+				includeContent: true,
+				// Must be set for most source map verifiers to work
+				source: sourceFileName || filename,
+				file: posix.basename(sourceFileName || filename || '')
 			});
 		}
 		return map;

packages/wmr/src/lib/normalize-options.js

@@ -21,7 +21,6 @@ export async function normalizeOptions(options, mode, configWatchFiles = []) {
 
 	options.root = options.cwd;
 
-	options.sourcemap = false;
 	options.minify = mode === 'build';
 	options.plugins = [];
 	options.output = [];

packages/wmr/src/lib/npm-middleware.js

@@ -142,6 +142,7 @@ async function bundleNpmModule(mod, { source, alias, cwd }) {
 			aliasPlugin({ alias, cwd }),
 			npmProviderPlugin,
 			processGlobalPlugin({
+				sourcemap: false,
 				NODE_ENV: 'development'
 			}),
 			commonjs({

packages/wmr/src/lib/plugins.js

@@ -53,21 +53,22 @@ export function getPlugins(options) {
 			production
 		}),
 		// Transpile import assertion syntax to WMR prefixes
-		importAssertionPlugin(),
+		importAssertionPlugin({ sourcemap }),
 		production &&
 			(dynamicImportVars.default || dynamicImportVars)({
 				include: /\.(m?jsx?|tsx?)$/,
 				exclude: /\/node_modules\//
 			}),
 		production && publicPathPlugin({ publicPath }),
 		sassPlugin({ production, sourcemap, root }),
-		wmrStylesPlugin({ hot: !production, root, production, alias }),
+		wmrStylesPlugin({ hot: !production, root, production, alias, sourcemap }),
 		processGlobalPlugin({
+			sourcemap,
 			env,
 			NODE_ENV: production ? 'production' : 'development'
 		}),
-		htmPlugin({ production }),
-		wmrPlugin({ hot: !production, preact: features.preact }),
+		htmPlugin({ production, sourcemap: options.sourcemap }),
+		wmrPlugin({ hot: !production, preact: features.preact, sourcemap: options.sourcemap }),
 		fastCjsPlugin({
 			// Only transpile CommonJS in node_modules and explicit .cjs files:
 			include: /(^npm\/|[/\\]node_modules[/\\]|\.cjs$)/

packages/wmr/src/lib/rollup-plugin-container.js

@@ -3,7 +3,8 @@ import { createHash } from 'crypto';
 import { promises as fs } from 'fs';
 import * as acorn from 'acorn';
 import * as kl from 'kolorist';
-import { debug, formatResolved, formatPath } from './output-utils.js';
+import { debug, formatResolved, formatPath, hasDebugFlag } from './output-utils.js';
+import { mergeSourceMaps } from './sourcemap.js';
 
 // Rollup respects "module", Node 14 doesn't.
 const cjsDefault = m => ('default' in m ? m.default : m);
@@ -45,7 +46,7 @@ function identifierPair(id, importer) {
 
 /**
  * @param {Plugin[]} plugins
- * @param {import('rollup').InputOptions & PluginContainerOptions} [opts]
+ * @param {import('rollup').InputOptions & PluginContainerOptions & {sourcemap?: boolean}} [opts]
  */
 export function createPluginContainer(plugins, opts = {}) {
 	if (!Array.isArray(plugins)) plugins = [plugins];
@@ -279,19 +280,42 @@ export function createPluginContainer(plugins, opts = {}) {
 		 * @param {string} id
 		 */
 		async transform(code, id) {
+			/** @type {import('./sourcemap.js').SourceMap[]} */
+			const sourceMaps = [];
+
 			for (plugin of plugins) {
 				if (!plugin.transform) continue;
 				const result = await plugin.transform.call(ctx, code, id);
 				if (!result) continue;
 
 				logTransform(`${kl.dim(formatPath(id))} [${plugin.name}]`);
 				if (typeof result === 'object') {
+					if (result.map) {
+						// Normalize source map sources URLs for the browser
+						result.map.sources = result.map.sources.map(s => {
+							if (typeof s === 'string') {
+								return `/${posix.normalize(s)}`;
+							} else if (hasDebugFlag()) {
+								logTransform(kl.yellow(`Invalid source map returned by plugin `) + kl.magenta(plugin.name));
+							}
+
+							return s;
+						});
+
+						sourceMaps.push(result.map);
+					} else if (opts.sourcemap && result.code !== code) {
+						logTransform(kl.yellow(`Missing sourcemap result in transform() method of `) + kl.magenta(plugin.name));
+					}
+
 					code = result.code;
 				} else {
+					if (opts.sourcemap && code !== result) {
+						logTransform(kl.yellow(`Missing sourcemap result in transform() method of `) + kl.magenta(plugin.name));
+					}
 					code = result;
 				}
 			}
-			return code;
+			return { code, map: sourceMaps.length ? mergeSourceMaps(sourceMaps) : null };
 		},
 
 		/**

packages/wmr/src/lib/sourcemap.js

@@ -0,0 +1,185 @@
+import { encode, decode } from 'sourcemap-codec';
+
+/**
+ * @typedef {{ version: number, file?: string, sourceRoot?: string, sources: string[], sourcesContent: Array<string | null>, names: string[], mappings: string}} SourceMap
+ */
+
+/**
+ *
+ * @param {SourceMap[]} maps
+ * @param {import('sourcemap-codec').SourceMapMappings[]} parsedMappings
+ * @param {Map<string, { index: number, content: string | null}>} sourcesCache
+ * @param {number} source
+ * @param {number} line
+ * @param {number} column
+ * @returns {{ line: number, column: number, source: number }}
+ */
+function getOriginalPosition(maps, parsedMappings, sourcesCache, source, line, column) {
+	let originalLine = line;
+	let originalColumn = column;
+	let originalSource = source;
+
+	// Traverse maps backwards, but skip last map as that's the
+	// one we requested the position from
+	for (let i = parsedMappings.length - 1; i >= 0; i--) {
+		const map = parsedMappings[i];
+
+		const segments = map[originalLine];
+		if (!segments) break;
+
+		// TODO: Binary search
+		let lastFound;
+		for (let j = 0; j < segments.length; j++) {
+			const segment = segments[j];
+			if (segment[0] === originalColumn) {
+				if (segment.length === 1) {
+					break;
+				}
+
+				const sourceName = maps[i].sources[segment[1]];
+				const mappedName = sourcesCache.get(sourceName);
+				if (!mappedName) {
+					originalSource = 0;
+				} else {
+					originalSource = mappedName.index;
+				}
+
+				originalLine = segment[2];
+				originalColumn = segment[3];
+				break;
+			} else if (segment[0] < originalColumn) {
+				lastFound = segment;
+			}
+		}
+
+		if (lastFound !== undefined) {
+			if (lastFound.length === 1) {
+				break;
+			}
+
+			const sourceName = maps[i].sources[lastFound[1]];
+			const mappedName = sourcesCache.get(sourceName);
+			if (!mappedName) {
+				originalSource = 0;
+			} else {
+				originalSource = mappedName.index;
+			}
+
+			originalLine = lastFound[2];
+			originalColumn = lastFound[3] + (originalColumn - lastFound[3]);
+		}
+	}
+
+	return {
+		line: originalLine,
+		column: originalColumn,
+		source: originalSource
+	};
+}
+
+/**
+ * Combine an array of sourcemaps into one
+ * @param {SourceMap[]} sourceMaps
+ * @returns {SourceMap}
+ */
+export function mergeSourceMaps(sourceMaps) {
+	let file;
+	let sourceRoot;
+
+	/** @type {import('sourcemap-codec').SourceMapMappings[]} */
+	const parsedMappings = [];
+
+	/** @type {Map<string, number>} */
+	const names = new Map();
+
+	/** @type {Map<string, { index: number, content: string | null}>} */
+	const sourcesCache = new Map();
+
+	for (let i = 0; i < sourceMaps.length; i++) {
+		const map = sourceMaps[i];
+
+		if (map.file) {
+			file = map.file;
+		}
+
+		if (map.sourceRoot) {
+			sourceRoot = map.sourceRoot;
+		}
+
+		for (let j = 0; j < map.sources.length; j++) {
+			const source = map.sources[j];
+			if (!sourcesCache.has(source)) {
+				sourcesCache.set(source, { index: sourcesCache.size, content: map.sourcesContent[j] || null });
+			}
+		}
+
+		for (let j = 0; j < map.names.length; j++) {
+			const name = map.names[j];
+			if (!names.has(name)) {
+				names.set(name, names.size - 1);
+			}
+		}
+
+		// Merge mappings
+		parsedMappings.push(decode(map.mappings));
+	}
+
+	const sources = [];
+	const sourcesContent = [];
+	for (const [key, value] of sourcesCache.entries()) {
+		sources.push(key);
+		sourcesContent.push(value.content);
+	}
+
+	/** @type {import('sourcemap-codec').SourceMapMappings} */
+	const outMappings = [];
+	const lastMap = parsedMappings[parsedMappings.length - 1];
+
+	// Loop over the mappings of the last source map and retrieve
+	// original position for each mapping segment
+	for (let i = 0; i < lastMap.length; i++) {
+		const line = lastMap[i];
+
+		/** @type {import('sourcemap-codec').SourceMapSegment[]} */
+		const rewrittenSegments = [];
+		for (let j = 0; j < line.length; j++) {
+			const segment = line[j];
+
+			if (segment.length === 4) {
+				const original = getOriginalPosition(
+					sourceMaps,
+					parsedMappings,
+					sourcesCache,
+					segment[1],
+					segment[2],
+					segment[3]
+				);
+				rewrittenSegments.push([segment[0], original.source, original.line, original.column]);
+			} else if (segment.length === 5) {
+				const original = getOriginalPosition(
+					sourceMaps,
+					parsedMappings,
+					sourcesCache,
+					segment[1],
+					segment[2],
+					segment[3]
+				);
+				rewrittenSegments.push([segment[0], original.source, original.line, original.column, segment[4]]);
+			} else {
+				rewrittenSegments.push(segment);
+			}
+		}
+
+		outMappings.push(rewrittenSegments);
+	}
+
+	return {
+		version: 3,
+		file,
+		names: Array.from(names.keys()),
+		sourceRoot,
+		sources,
+		sourcesContent,
+		mappings: encode(outMappings)
+	};
+}