From 511460e67495b63dfae7af37f460b348ace37dc5 Mon Sep 17 00:00:00 2001 From: sapphi-red Date: Wed, 21 Jun 2023 20:06:17 +0900 Subject: [PATCH 01/16] perf: use thread for preprocessors --- packages/vite/LICENSE.md | 29 ---- packages/vite/src/node/okie.ts | 188 +++++++++++++++++++++++ packages/vite/src/node/plugins/css.ts | 164 ++++++++++++++------ packages/vite/src/node/plugins/terser.ts | 2 +- 4 files changed, 309 insertions(+), 74 deletions(-) create mode 100644 packages/vite/src/node/okie.ts diff --git a/packages/vite/LICENSE.md b/packages/vite/LICENSE.md index 6021274c459860..0fed7f357fb40a 100644 --- a/packages/vite/LICENSE.md +++ b/packages/vite/LICENSE.md @@ -2201,35 +2201,6 @@ Repository: sindresorhus/object-assign --------------------------------------- -## okie -License: MIT -By: Evan You -Repository: git+https://github.com/yyx990803/okie.git - -> MIT License -> -> Copyright (c) 2020-present, Yuxi (Evan) You -> -> Permission is hereby granted, free of charge, to any person obtaining a copy -> of this software and associated documentation files (the "Software"), to deal -> in the Software without restriction, including without limitation the rights -> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -> copies of the Software, and to permit persons to whom the Software is -> furnished to do so, subject to the following conditions: -> -> The above copyright notice and this permission notice shall be included in all -> copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -> SOFTWARE. - ---------------------------------------- - ## on-finished License: MIT By: Douglas Christopher Wilson, Jonathan Ong diff --git a/packages/vite/src/node/okie.ts b/packages/vite/src/node/okie.ts new file mode 100644 index 00000000000000..6dcf9de27494d7 --- /dev/null +++ b/packages/vite/src/node/okie.ts @@ -0,0 +1,188 @@ +import os from 'node:os' +import { Worker as _Worker } from 'node:worker_threads' + +interface NodeWorker extends _Worker { + currentResolve: ((value: any) => void) | null + currentReject: ((err: Error) => void) | null +} + +export interface Options { + max?: number + parentFunctions?: Record Promise> +} + +export class Worker { + private code: string + private parentFunctions: Record Promise> + private max: number + private pool: NodeWorker[] + private idlePool: NodeWorker[] + private queue: [(worker: NodeWorker) => void, (err: Error) => void][] + + constructor( + fn: (...args: Args) => Promise | Ret, + options: Options = {}, + ) { + this.code = genWorkerCode(fn, options.parentFunctions ?? {}) + this.parentFunctions = options.parentFunctions ?? {} + this.max = options.max || Math.max(1, os.cpus().length - 1) + this.pool = [] + this.idlePool = [] + this.queue = [] + } + + async run(...args: Args): Promise { + const worker = await this._getAvailableWorker() + return new Promise((resolve, reject) => { + worker.currentResolve = resolve + worker.currentReject = reject + worker.postMessage({ type: 'run', args }) + }) + } + + stop(): void { + this.pool.forEach((w) => w.unref()) + this.queue.forEach(([_, reject]) => + reject( + new Error('Main worker pool stopped before a worker was available.'), + ), + ) + this.pool = [] + this.idlePool = [] + this.queue = [] + } + + private async _getAvailableWorker(): Promise { + // has idle one? + if (this.idlePool.length) { + return this.idlePool.shift()! + } + + // can spawn more? + if (this.pool.length < this.max) { + const worker = new _Worker(this.code, { eval: true }) as NodeWorker + + worker.on('message', async (args) => { + if (args.type === 'run') { + if ('result' in args) { + worker.currentResolve && worker.currentResolve(args.result) + worker.currentResolve = null + this._assignDoneWorker(worker) + } else { + worker.currentReject && worker.currentReject(args.error) + worker.currentReject = null + } + } else if (args.type === 'parentFunction') { + if (!(args.name in this.parentFunctions)) { + throw new Error( + `Parent function ${JSON.stringify( + args.name, + )} was not passed to options but was called.`, + ) + } + + try { + const result = await this.parentFunctions[args.name](...args.args) + worker.postMessage({ type: 'parentFunction', id: args.id, result }) + } catch (e) { + worker.postMessage({ + type: 'parentFunction', + id: args.id, + error: e, + }) + } + } + }) + + worker.on('error', (err) => { + worker.currentReject && worker.currentReject(err) + worker.currentReject = null + }) + + worker.on('exit', (code) => { + const i = this.pool.indexOf(worker) + if (i > -1) this.pool.splice(i, 1) + if (code !== 0 && worker.currentReject) { + worker.currentReject( + new Error(`Worker stopped with non-0 exit code ${code}`), + ) + worker.currentReject = null + } + }) + + this.pool.push(worker) + return worker + } + + // no one is available, we have to wait + let resolve: (worker: NodeWorker) => void + let reject: (err: Error) => any + const onWorkerAvailablePromise = new Promise((r, rj) => { + resolve = r + reject = rj + }) + this.queue.push([resolve!, reject!]) + return onWorkerAvailablePromise + } + + private _assignDoneWorker(worker: NodeWorker) { + // someone's waiting already? + if (this.queue.length) { + const [resolve] = this.queue.shift()! + resolve(worker) + return + } + // take a rest. + this.idlePool.push(worker) + } +} + +function genWorkerCode(fn: Function, parentFunctions: Record) { + return ` +let id = 0 +const parentFunctionResolvers = new Map() +const parentFunctionCall = (key) => async (...args) => { + id++ + let resolve, reject + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + parentFunctionResolvers.set(id, { resolve, reject }) + + parentPort.postMessage({ type: 'parentFunction', id, name: key, args }) + return await promise +} + +const doWork = (() => { + ${Object.keys(parentFunctions) + .map((key) => `const ${key} = parentFunctionCall(${JSON.stringify(key)});`) + .join('\n')} + return ${fn.toString()} +})() + +const { parentPort } = require('worker_threads') + +parentPort.on('message', async (args) => { + if (args.type === 'run') { + try { + const res = await doWork(...args.args) + parentPort.postMessage({ type: 'run', result: res }) + } catch (e) { + parentPort.postMessage({ type: 'run', error: e }) + } + } else if (args.type === 'parentFunction') { + if (parentFunctionResolvers.has(id)) { + const { resolve, reject } = parentFunctionResolvers.get(id) + parentFunctionResolvers.delete(id) + + if ('result' in args) { + resolve(args.result) + } else { + reject(args.error) + } + } + } +}) + ` +} diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 9aa3abf5c2da68..2ac883da1fcab7 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -58,6 +58,7 @@ import { stripBomTag, } from '../utils' import type { Logger } from '../logger' +import { Worker } from '../okie' import { addToHTMLProxyTransformResult } from './html' import { assetUrlRE, @@ -362,6 +363,9 @@ export function cssPlugin(config: ResolvedConfig): Plugin { map, } }, + buildEnd() { + scssWorker.stop() + }, } } @@ -1704,6 +1708,36 @@ function loadPreprocessor( } } +const loadedPreprocessorPath: Partial< + Record +> = {} + +function loadPreprocessorPath( + lang: PreprocessLang | PostCssDialectLang, + root: string, +): string { + const cached = loadedPreprocessorPath[lang] + if (cached) { + return cached + } + try { + const resolved = requireResolveFromRootWithFallback(root, lang) + return (loadedPreprocessorPath[lang] = resolved) + } catch (e) { + if (e.code === 'MODULE_NOT_FOUND') { + throw new Error( + `Preprocessor dependency "${lang}" not found. Did you install it?`, + ) + } else { + const message = new Error( + `Preprocessor dependency "${lang}" failed to load:\n${e.message}`, + ) + message.stack = e.stack + '\n' + message.stack + throw message + } + } +} + declare const window: unknown | undefined declare const location: { href: string } | undefined @@ -1742,6 +1776,89 @@ function fixScssBugImportValue( return data } +let scssWorker: ReturnType +const makeScssWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { + const internalImporter = async ( + url: string, + importer: string, + filename: string, + ) => { + importer = cleanScssBugUrl(importer) + const resolved = await resolvers.sass(url, importer) + if (resolved) { + try { + const data = await rebaseUrls(resolved, filename, alias, '$') + return fixScssBugImportValue(data) + } catch (data) { + return data + } + } else { + return null + } + } + + const worker = new Worker( + async ( + sassPath: string, + data: string, + options: SassStylePreprocessorOptions, + ) => { + // eslint-disable-next-line no-restricted-globals + const sass: typeof Sass = require(sassPath) + // eslint-disable-next-line no-restricted-globals + const path = require('node:path') + + // NOTE: `sass` always runs it's own importer first, and only falls back to + // the `importer` option when it can't resolve a path + const _internalImporter: Sass.Importer = (url, importer, done) => { + internalImporter(url, importer, options.filename).then((data) => + done?.(data), + ) + } + const importer = [_internalImporter] + if (options.importer) { + Array.isArray(options.importer) + ? importer.unshift(...options.importer) + : importer.unshift(options.importer) + } + + const finalOptions: Sass.Options = { + ...options, + data, + file: options.filename, + outFile: options.filename, + importer, + ...(options.enableSourcemap + ? { + sourceMap: true, + omitSourceMapUrl: true, + sourceMapRoot: path.dirname(options.filename), + } + : {}), + } + return new Promise<{ + css: string + map?: string | undefined + stats: Sass.Result['stats'] + }>((resolve, reject) => { + sass.render(finalOptions, (err, res) => { + if (err) { + reject(err) + } else { + resolve({ + css: res.css.toString(), + map: res.map?.toString(), + stats: res.stats, + }) + } + }) + }) + }, + { parentFunctions: { internalImporter } }, + ) + return worker +} + // .scss/.sass processor const scss: SassStylePreprocessor = async ( source, @@ -1749,27 +1866,8 @@ const scss: SassStylePreprocessor = async ( options, resolvers, ) => { - const render = loadPreprocessor(PreprocessLang.sass, root).render - // NOTE: `sass` always runs it's own importer first, and only falls back to - // the `importer` option when it can't resolve a path - const internalImporter: Sass.Importer = (url, importer, done) => { - importer = cleanScssBugUrl(importer) - resolvers.sass(url, importer).then((resolved) => { - if (resolved) { - rebaseUrls(resolved, options.filename, options.alias, '$') - .then((data) => done?.(fixScssBugImportValue(data))) - .catch((data) => done?.(data)) - } else { - done?.(null) - } - }) - } - const importer = [internalImporter] - if (options.importer) { - Array.isArray(options.importer) - ? importer.unshift(...options.importer) - : importer.unshift(options.importer) - } + const sassPath = loadPreprocessorPath(PreprocessLang.sass, root) + scssWorker ||= makeScssWorker(resolvers, options.alias) const { content: data, map: additionalMap } = await getSource( source, @@ -1777,31 +1875,9 @@ const scss: SassStylePreprocessor = async ( options.additionalData, options.enableSourcemap, ) - const finalOptions: Sass.Options = { - ...options, - data, - file: options.filename, - outFile: options.filename, - importer, - ...(options.enableSourcemap - ? { - sourceMap: true, - omitSourceMapUrl: true, - sourceMapRoot: path.dirname(options.filename), - } - : {}), - } try { - const result = await new Promise((resolve, reject) => { - render(finalOptions, (err, res) => { - if (err) { - reject(err) - } else { - resolve(res) - } - }) - }) + const result = await scssWorker.run(sassPath, data, options) const deps = result.stats.includedFiles.map((f) => cleanScssBugUrl(f)) const map: ExistingRawSourceMap | undefined = result.map ? JSON.parse(result.map.toString()) diff --git a/packages/vite/src/node/plugins/terser.ts b/packages/vite/src/node/plugins/terser.ts index 40f28cb9daccaf..03f12ff464e968 100644 --- a/packages/vite/src/node/plugins/terser.ts +++ b/packages/vite/src/node/plugins/terser.ts @@ -1,5 +1,5 @@ -import { Worker } from 'okie' import type { Terser } from 'dep-types/terser' +import { Worker } from '../okie' import type { Plugin } from '../plugin' import type { ResolvedConfig } from '..' import { requireResolveFromRootWithFallback } from '../utils' From 6d4580146ee64635d4fa3a06d43f1bdeaddc82db Mon Sep 17 00:00:00 2001 From: sapphi-red Date: Sun, 13 Aug 2023 23:43:32 +0900 Subject: [PATCH 02/16] wip: implement all preprocessors --- packages/vite/src/node/index.ts | 6 +- packages/vite/src/node/okie.ts | 71 +- packages/vite/src/node/plugins/css.ts | 822 +++++++++++++---------- packages/vite/src/node/plugins/terser.ts | 21 +- 4 files changed, 567 insertions(+), 353 deletions(-) diff --git a/packages/vite/src/node/index.ts b/packages/vite/src/node/index.ts index ffa1d4ecd83a49..c15cba96e8aa44 100644 --- a/packages/vite/src/node/index.ts +++ b/packages/vite/src/node/index.ts @@ -6,7 +6,11 @@ export { createServer } from './server' export { preview } from './preview' export { build } from './build' export { optimizeDeps } from './optimizer' -export { formatPostcssSourceMap, preprocessCSS } from './plugins/css' +export { + formatPostcssSourceMap, + createCSSPreprocessor, + preprocessCSS, +} from './plugins/css' export { transformWithEsbuild } from './plugins/esbuild' export { resolvePackageEntry } from './plugins/resolve' export { resolvePackageData } from './packages' diff --git a/packages/vite/src/node/okie.ts b/packages/vite/src/node/okie.ts index 6dcf9de27494d7..0125f4656ff9d0 100644 --- a/packages/vite/src/node/okie.ts +++ b/packages/vite/src/node/okie.ts @@ -1,3 +1,4 @@ +import { createRequire } from 'node:module' import os from 'node:os' import { Worker as _Worker } from 'node:worker_threads' @@ -20,7 +21,7 @@ export class Worker { private queue: [(worker: NodeWorker) => void, (err: Error) => void][] constructor( - fn: (...args: Args) => Promise | Ret, + fn: () => (...args: Args) => Promise | Ret, options: Options = {}, ) { this.code = genWorkerCode(fn, options.parentFunctions ?? {}) @@ -158,7 +159,7 @@ const doWork = (() => { ${Object.keys(parentFunctions) .map((key) => `const ${key} = parentFunctionCall(${JSON.stringify(key)});`) .join('\n')} - return ${fn.toString()} + return (${fn.toString()})() })() const { parentPort } = require('worker_threads') @@ -172,6 +173,7 @@ parentPort.on('message', async (args) => { parentPort.postMessage({ type: 'run', error: e }) } } else if (args.type === 'parentFunction') { + const id = args.id if (parentFunctionResolvers.has(id)) { const { resolve, reject } = parentFunctionResolvers.get(id) parentFunctionResolvers.delete(id) @@ -186,3 +188,68 @@ parentPort.on('message', async (args) => { }) ` } + +class FakeWorker { + private fn: (...args: Args) => Promise + + constructor( + fn: () => (...args: Args) => Promise | Ret, + options: Options = {}, + ) { + const argsAndCode = genFakeWorkerArgsAndCode( + fn, + options.parentFunctions ?? {}, + ) + const require = createRequire(import.meta.url) + this.fn = new Function(...argsAndCode)(require, options.parentFunctions) + } + + async run(...args: Args): Promise { + return this.fn(...args) + } + + stop(): void { + /* no-op */ + } +} + +function genFakeWorkerArgsAndCode( + fn: Function, + parentFunctions: Record, +) { + return [ + 'require', + 'parentFunctions', + ` +${Object.keys(parentFunctions) + .map((key) => `const ${key} = parentFunctions[${JSON.stringify(key)}];`) + .join('\n')} +return (${fn.toString()})() + `, + ] +} + +export class WorkerWithFallback { + private _realWorker: Worker + private _fakeWorker: FakeWorker + private _shouldUseFake: (...args: Args) => boolean + + constructor( + fn: () => (...args: Args) => Promise | Ret, + options: Options & { shouldUseFake: (...args: Args) => boolean }, + ) { + this._realWorker = new Worker(fn, options) + this._fakeWorker = new FakeWorker(fn, options) + this._shouldUseFake = options.shouldUseFake + } + + async run(...args: Args): Promise { + const useFake = this._shouldUseFake(...args) + return this[useFake ? '_fakeWorker' : '_realWorker'].run(...args) + } + + stop(): void { + this._realWorker.stop() + this._fakeWorker.stop() + } +} diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 2ac883da1fcab7..c51b2a1c7aceb9 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -58,7 +58,7 @@ import { stripBomTag, } from '../utils' import type { Logger } from '../logger' -import { Worker } from '../okie' +import { WorkerWithFallback } from '../okie' import { addToHTMLProxyTransformResult } from './html' import { assetUrlRE, @@ -232,6 +232,8 @@ export function cssPlugin(config: ResolvedConfig): Plugin { extensions: [], }) + const preprocessorWorkerController = createPreprocessorWorkerController() + // warm up cache for resolved postcss config if (config.css?.transformer !== 'lightningcss') { resolvePostcssConfig(config) @@ -299,7 +301,13 @@ export function cssPlugin(config: ResolvedConfig): Plugin { modules, deps, map, - } = await compileCSS(id, raw, config, urlReplacer) + } = await compileCSS( + id, + raw, + config, + preprocessorWorkerController, + urlReplacer, + ) if (modules) { moduleCache.set(id, modules) } @@ -364,7 +372,7 @@ export function cssPlugin(config: ResolvedConfig): Plugin { } }, buildEnd() { - scssWorker.stop() + preprocessorWorkerController.close() }, } } @@ -869,11 +877,12 @@ async function compileCSSPreprocessors( lang: PreprocessLang, code: string, config: ResolvedConfig, + workerController: PreprocessorWorkerController, ): Promise<{ code: string; map?: ExistingRawSourceMap; deps?: Set }> { const { preprocessorOptions, devSourcemap } = config.css ?? {} const atImportResolvers = getAtImportResolvers(config) - const preProcessor = preProcessors[lang] + const preProcessor = workerController[lang] let opts = (preprocessorOptions && preprocessorOptions[lang]) || {} // support @import from node dependencies by default switch (lang) { @@ -947,6 +956,7 @@ async function compileCSS( id: string, code: string, config: ResolvedConfig, + workerController: PreprocessorWorkerController, urlReplacer?: CssUrlReplacer, ): Promise<{ code: string @@ -990,6 +1000,7 @@ async function compileCSS( lang, code, config, + workerController, ) code = preprocessorResult.code preprocessorMap = preprocessorResult.map @@ -1038,7 +1049,13 @@ async function compileCSS( const code = await fs.promises.readFile(id, 'utf-8') const lang = id.match(CSS_LANGS_RE)?.[1] as CssLang | undefined if (isPreProcessor(lang)) { - const result = await compileCSSPreprocessors(id, lang, code, config) + const result = await compileCSSPreprocessors( + id, + lang, + code, + config, + workerController, + ) result.deps?.forEach((dep) => deps.add(dep)) // TODO: support source map return result.code @@ -1105,10 +1122,7 @@ async function compileCSS( // postcss is an unbundled dep and should be lazy imported postcssResult = await postcss.default(postcssPlugins).process(code, { ...postcssOptions, - parser: - lang === 'sss' - ? loadPreprocessor(PostCssDialectLang.sss, config.root) - : postcssOptions.parser, + parser: lang === 'sss' ? loadSss(config.root) : postcssOptions.parser, to: source, from: source, ...(devSourcemap @@ -1218,12 +1232,45 @@ export interface PreprocessCSSResult { /** * @experimental */ +export function createCSSPreprocessor(): { + process( + code: string, + filename: string, + config: ResolvedConfig, + ): Promise + close: () => void +} { + const preprocessorWorkerController = createPreprocessorWorkerController() + + return { + async process(code, filename, config) { + return await compileCSS( + filename, + code, + config, + preprocessorWorkerController, + ) + }, + close() { + preprocessorWorkerController.close() + }, + } +} + +/** + * @deprecated use createCSSPreprocessor instead + */ export async function preprocessCSS( code: string, filename: string, config: ResolvedConfig, ): Promise { - return await compileCSS(filename, code, config) + const p = createCSSPreprocessor() + try { + return p.process(code, filename, config) + } finally { + p.close() + } } const postcssReturnsVirtualFilesRE = /^<.+>$/ @@ -1635,26 +1682,35 @@ type StylusStylePreprocessorOptions = StylePreprocessorOptions & { define?: Record } -type StylePreprocessor = ( - source: string, - root: string, - options: StylePreprocessorOptions, - resolvers: CSSAtImportResolvers, -) => StylePreprocessorResults | Promise +type StylePreprocessor = { + process: ( + source: string, + root: string, + options: StylePreprocessorOptions, + resolvers: CSSAtImportResolvers, + ) => StylePreprocessorResults | Promise + close: () => void +} -type SassStylePreprocessor = ( - source: string, - root: string, - options: SassStylePreprocessorOptions, - resolvers: CSSAtImportResolvers, -) => StylePreprocessorResults | Promise +type SassStylePreprocessor = { + process: ( + source: string, + root: string, + options: SassStylePreprocessorOptions, + resolvers: CSSAtImportResolvers, + ) => StylePreprocessorResults | Promise + close: () => void +} -type StylusStylePreprocessor = ( - source: string, - root: string, - options: StylusStylePreprocessorOptions, - resolvers: CSSAtImportResolvers, -) => StylePreprocessorResults | Promise +type StylusStylePreprocessor = { + process: ( + source: string, + root: string, + options: StylusStylePreprocessorOptions, + resolvers: CSSAtImportResolvers, + ) => StylePreprocessorResults | Promise + close: () => void +} export interface StylePreprocessorResults { code: string @@ -1664,50 +1720,6 @@ export interface StylePreprocessorResults { deps: string[] } -const loadedPreprocessors: Partial< - Record -> = {} - -// TODO: use dynamic import -const _require = createRequire(import.meta.url) - -function loadPreprocessor(lang: PreprocessLang.scss, root: string): typeof Sass -function loadPreprocessor(lang: PreprocessLang.sass, root: string): typeof Sass -function loadPreprocessor(lang: PreprocessLang.less, root: string): typeof Less -function loadPreprocessor( - lang: PreprocessLang.stylus, - root: string, -): typeof Stylus -function loadPreprocessor( - lang: PostCssDialectLang.sss, - root: string, -): PostCSS.Parser -function loadPreprocessor( - lang: PreprocessLang | PostCssDialectLang, - root: string, -): any { - if (lang in loadedPreprocessors) { - return loadedPreprocessors[lang] - } - try { - const resolved = requireResolveFromRootWithFallback(root, lang) - return (loadedPreprocessors[lang] = _require(resolved)) - } catch (e) { - if (e.code === 'MODULE_NOT_FOUND') { - const installCommand = getPackageManagerCommand('install') - throw new Error( - `Preprocessor dependency "${lang}" not found. Did you install it? Try \`${installCommand} -D ${lang}\`.`, - ) - } else { - const message = new Error( - `Preprocessor dependency "${lang}" failed to load:\n${e.message}`, - ) - message.stack = e.stack + '\n' + message.stack - throw message - } - } -} - const loadedPreprocessorPath: Partial< Record > = {} @@ -1724,9 +1736,10 @@ function loadPreprocessorPath( const resolved = requireResolveFromRootWithFallback(root, lang) return (loadedPreprocessorPath[lang] = resolved) } catch (e) { + const installCommand = getPackageManagerCommand('install') if (e.code === 'MODULE_NOT_FOUND') { throw new Error( - `Preprocessor dependency "${lang}" not found. Did you install it?`, + `Preprocessor dependency "${lang}" not found. Did you install it? Try \`${installCommand} -D ${lang}\`.`, ) } else { const message = new Error( @@ -1738,6 +1751,15 @@ function loadPreprocessorPath( } } +let cachedSss: any +function loadSss(root: string) { + if (cachedSss) return cachedSss + + const sssPath = loadPreprocessorPath(PostCssDialectLang.sss, root) + cachedSss = createRequire(import.meta.url)(sssPath) + return cachedSss +} + declare const window: unknown | undefined declare const location: { href: string } | undefined @@ -1776,7 +1798,7 @@ function fixScssBugImportValue( return data } -let scssWorker: ReturnType +// .scss/.sass processor const makeScssWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { const internalImporter = async ( url: string, @@ -1797,117 +1819,125 @@ const makeScssWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { } } - const worker = new Worker( - async ( - sassPath: string, - data: string, - options: SassStylePreprocessorOptions, - ) => { - // eslint-disable-next-line no-restricted-globals - const sass: typeof Sass = require(sassPath) - // eslint-disable-next-line no-restricted-globals - const path = require('node:path') - - // NOTE: `sass` always runs it's own importer first, and only falls back to - // the `importer` option when it can't resolve a path - const _internalImporter: Sass.Importer = (url, importer, done) => { - internalImporter(url, importer, options.filename).then((data) => - done?.(data), - ) - } - const importer = [_internalImporter] - if (options.importer) { - Array.isArray(options.importer) - ? importer.unshift(...options.importer) - : importer.unshift(options.importer) - } + const worker = new WorkerWithFallback( + () => + async ( + sassPath: string, + data: string, + options: SassStylePreprocessorOptions, + ) => { + // eslint-disable-next-line no-restricted-globals + const sass: typeof Sass = require(sassPath) + // eslint-disable-next-line no-restricted-globals + const path = require('node:path') + + // NOTE: `sass` always runs it's own importer first, and only falls back to + // the `importer` option when it can't resolve a path + const _internalImporter: Sass.Importer = (url, importer, done) => { + internalImporter(url, importer, options.filename).then( + (data) => done?.(data), + ) + } + const importer = [_internalImporter] + if (options.importer) { + Array.isArray(options.importer) + ? importer.unshift(...options.importer) + : importer.unshift(options.importer) + } - const finalOptions: Sass.Options = { - ...options, - data, - file: options.filename, - outFile: options.filename, - importer, - ...(options.enableSourcemap - ? { - sourceMap: true, - omitSourceMapUrl: true, - sourceMapRoot: path.dirname(options.filename), + const finalOptions: Sass.Options = { + ...options, + data, + file: options.filename, + outFile: options.filename, + importer, + ...(options.enableSourcemap + ? { + sourceMap: true, + omitSourceMapUrl: true, + sourceMapRoot: path.dirname(options.filename), + } + : {}), + } + return new Promise<{ + css: string + map?: string | undefined + stats: Sass.Result['stats'] + }>((resolve, reject) => { + sass.render(finalOptions, (err, res) => { + if (err) { + reject(err) + } else { + resolve({ + css: res.css.toString(), + map: res.map?.toString(), + stats: res.stats, + }) } - : {}), - } - return new Promise<{ - css: string - map?: string | undefined - stats: Sass.Result['stats'] - }>((resolve, reject) => { - sass.render(finalOptions, (err, res) => { - if (err) { - reject(err) - } else { - resolve({ - css: res.css.toString(), - map: res.map?.toString(), - stats: res.stats, - }) - } + }) }) - }) + }, + { + parentFunctions: { internalImporter }, + shouldUseFake(_sassPath, _data, options) { + return !!( + (options.functions && Object.keys(options.functions).length > 0) || + (options.importer && + (!Array.isArray(options.importer) || options.importer.length > 0)) + ) + }, }, - { parentFunctions: { internalImporter } }, ) return worker } -// .scss/.sass processor -const scss: SassStylePreprocessor = async ( - source, - root, - options, - resolvers, -) => { - const sassPath = loadPreprocessorPath(PreprocessLang.sass, root) - scssWorker ||= makeScssWorker(resolvers, options.alias) - - const { content: data, map: additionalMap } = await getSource( - source, - options.filename, - options.additionalData, - options.enableSourcemap, - ) +const scssProcessor = (): SassStylePreprocessor => { + const workerMap = new Map>() - try { - const result = await scssWorker.run(sassPath, data, options) - const deps = result.stats.includedFiles.map((f) => cleanScssBugUrl(f)) - const map: ExistingRawSourceMap | undefined = result.map - ? JSON.parse(result.map.toString()) - : undefined + return { + close() { + for (const worker of workerMap.values()) { + worker.stop() + } + }, + async process(source, root, options, resolvers) { + const sassPath = loadPreprocessorPath(PreprocessLang.sass, root) - return { - code: result.css.toString(), - map, - additionalMap, - deps, - } - } catch (e) { - // normalize SASS error - e.message = `[sass] ${e.message}` - e.id = e.file - e.frame = e.formatted - return { code: '', error: e, deps: [] } - } -} + if (!workerMap.has(options.alias)) { + workerMap.set(options.alias, makeScssWorker(resolvers, options.alias)) + } + const worker = workerMap.get(options.alias)! -const sass: SassStylePreprocessor = (source, root, options, aliasResolver) => - scss( - source, - root, - { - ...options, - indentedSyntax: true, + const { content: data, map: additionalMap } = await getSource( + source, + options.filename, + options.additionalData, + options.enableSourcemap, + ) + + try { + const result = await worker.run(sassPath, data, options) + const deps = result.stats.includedFiles.map((f) => cleanScssBugUrl(f)) + const map: ExistingRawSourceMap | undefined = result.map + ? JSON.parse(result.map.toString()) + : undefined + + return { + code: result.css.toString(), + map, + additionalMap, + deps, + } + } catch (e) { + // normalize SASS error + e.message = `[sass] ${e.message}` + e.id = e.file + e.frame = e.formatted + return { code: '', error: e, deps: [] } + } }, - aliasResolver, - ) + } +} /** * relative url() inside \@imported sass and less files must be rebased to use @@ -1977,187 +2007,265 @@ async function rebaseUrls( } // .less -const less: StylePreprocessor = async (source, root, options, resolvers) => { - const nodeLess = loadPreprocessor(PreprocessLang.less, root) - const viteResolverPlugin = createViteLessPlugin( - nodeLess, - options.filename, - options.alias, - resolvers, - ) - const { content, map: additionalMap } = await getSource( - source, - options.filename, - options.additionalData, - options.enableSourcemap, - ) +const makeLessWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { + const viteLessResolve = async ( + filename: string, + dir: string, + rootFile: string, + ) => { + const resolved = await resolvers.less(filename, path.join(dir, '*')) + if (!resolved) return undefined - let result: Less.RenderOutput | undefined - try { - result = await nodeLess.render(content, { - ...options, - plugins: [viteResolverPlugin, ...(options.plugins || [])], - ...(options.enableSourcemap - ? { - sourceMap: { - outputSourceFiles: true, - sourceMapFileInline: false, - }, - } - : {}), - }) - } catch (e) { - const error = e as Less.RenderError - // normalize error info - const normalizedError: RollupError = new Error( - `[less] ${error.message || error.type}`, - ) as RollupError - normalizedError.loc = { - file: error.filename || options.filename, - line: error.line, - column: error.column, + const result = await rebaseUrls(resolved, rootFile, alias, '@') + if (result) { + return { + resolved, + contents: 'contents' in result ? result.contents : undefined, + } } - return { code: '', error: normalizedError, deps: [] } - } - - const map: ExistingRawSourceMap = result.map && JSON.parse(result.map) - if (map) { - delete map.sourcesContent + return result } - return { - code: result.css.toString(), - map, - additionalMap, - deps: result.imports, - } -} - -/** - * Less manager, lazy initialized - */ -let ViteLessManager: any + const worker = new WorkerWithFallback( + () => { + // eslint-disable-next-line no-restricted-globals + const fsp = require('node:fs/promises') + // eslint-disable-next-line no-restricted-globals + const path = require('node:path') -function createViteLessPlugin( - less: typeof Less, - rootFile: string, - alias: Alias[], - resolvers: CSSAtImportResolvers, -): Less.Plugin { - if (!ViteLessManager) { - ViteLessManager = class ViteManager extends less.FileManager { - resolvers - rootFile - alias - constructor( + let ViteLessManager: any + const createViteLessPlugin = ( + less: typeof Less, rootFile: string, - resolvers: CSSAtImportResolvers, - alias: Alias[], - ) { - super() - this.rootFile = rootFile - this.resolvers = resolvers - this.alias = alias - } - override supports(filename: string) { - return !isExternalUrl(filename) - } - override supportsSync() { - return false - } - override async loadFile( - filename: string, - dir: string, - opts: any, - env: any, - ): Promise { - const resolved = await this.resolvers.less( - filename, - path.join(dir, '*'), - ) - if (resolved) { - const result = await rebaseUrls( - resolved, - this.rootFile, - this.alias, - '@', - ) - let contents: string - if (result && 'contents' in result) { - contents = result.contents - } else { - contents = await fsp.readFile(resolved, 'utf-8') + ): Less.Plugin => { + ViteLessManager ??= class ViteManager extends less.FileManager { + rootFile + constructor(rootFile: string) { + super() + this.rootFile = rootFile } - return { - filename: path.resolve(resolved), - contents, + override supports(filename: string) { + return !/^(?:https?:)?\/\//.test(filename) + } + override supportsSync() { + return false + } + override async loadFile( + filename: string, + dir: string, + opts: any, + env: any, + ): Promise { + const result = await viteLessResolve(filename, dir, this.rootFile) + if (result) { + return { + filename: path.resolve(result.resolved), + contents: + result.contents ?? + (await fsp.readFile(result.resolved, 'utf-8')), + } + } else { + return super.loadFile(filename, dir, opts, env) + } } - } else { - return super.loadFile(filename, dir, opts, env) + } + + return { + install(_, pluginManager) { + pluginManager.addFileManager(new ViteLessManager(rootFile)) + }, + minVersion: [3, 0, 0], } } - } - } + + return async ( + lessPath: string, + content: string, + options: StylePreprocessorOptions, + ) => { + // eslint-disable-next-line no-restricted-globals + const nodeLess: typeof Less = require(lessPath) + const viteResolverPlugin = createViteLessPlugin( + nodeLess, + options.filename, + ) + const result = await nodeLess.render(content, { + ...options, + plugins: [viteResolverPlugin, ...(options.plugins || [])], + ...(options.enableSourcemap + ? { + sourceMap: { + outputSourceFiles: true, + sourceMapFileInline: false, + }, + } + : {}), + }) + return result + } + }, + { + parentFunctions: { viteLessResolve }, + shouldUseFake(_lessPath, _content, options) { + return options.plugins?.length > 0 + }, + }, + ) + return worker +} + +const lessProcessor = (): StylePreprocessor => { + const workerMap = new Map>() return { - install(_, pluginManager) { - pluginManager.addFileManager( - new ViteLessManager(rootFile, resolvers, alias), + close() { + for (const worker of workerMap.values()) { + worker.stop() + } + }, + async process(source, root, options, resolvers) { + const lessPath = loadPreprocessorPath(PreprocessLang.less, root) + + if (!workerMap.has(options.alias)) { + workerMap.set(options.alias, makeLessWorker(resolvers, options.alias)) + } + const worker = workerMap.get(options.alias)! + + const { content, map: additionalMap } = await getSource( + source, + options.filename, + options.additionalData, + options.enableSourcemap, ) + + let result: Less.RenderOutput | undefined + try { + result = await worker.run(lessPath, content, options) + } catch (e) { + const error = e as Less.RenderError + // normalize error info + const normalizedError: RollupError = new Error( + `[less] ${error.message || error.type}`, + ) as RollupError + normalizedError.loc = { + file: error.filename || options.filename, + line: error.line, + column: error.column, + } + return { code: '', error: normalizedError, deps: [] } + } + + const map: ExistingRawSourceMap = result.map && JSON.parse(result.map) + if (map) { + delete map.sourcesContent + } + + return { + code: result.css.toString(), + map, + additionalMap, + deps: result.imports, + } }, - minVersion: [3, 0, 0], } } // .styl -const styl: StylusStylePreprocessor = async (source, root, options) => { - const nodeStylus = loadPreprocessor(PreprocessLang.stylus, root) - // Get source with preprocessor options.additionalData. Make sure a new line separator - // is added to avoid any render error, as added stylus content may not have semi-colon separators - const { content, map: additionalMap } = await getSource( - source, - options.filename, - options.additionalData, - options.enableSourcemap, - '\n', - ) - // Get preprocessor options.imports dependencies as stylus - // does not return them with its builtin `.deps()` method - const importsDeps = (options.imports ?? []).map((dep: string) => - path.resolve(dep), - ) - try { - const ref = nodeStylus(content, options) - if (options.define) { - for (const key in options.define) { - ref.define(key, options.define[key]) - } - } - if (options.enableSourcemap) { - ref.set('sourcemap', { - comment: false, - inline: false, - basePath: root, - }) - } +const makeStylWorker = () => { + const worker = new WorkerWithFallback( + () => { + return async ( + stylusPath: string, + content: string, + root: string, + options: StylePreprocessorOptions, + ) => { + // eslint-disable-next-line no-restricted-globals + const nodeStylus: typeof Stylus = require(stylusPath) + + const ref = nodeStylus(content, options) + if (options.define) { + for (const key in options.define) { + ref.define(key, options.define[key]) + } + } + if (options.enableSourcemap) { + ref.set('sourcemap', { + comment: false, + inline: false, + basePath: root, + }) + } - const result = ref.render() + return { + code: ref.render(), + // @ts-expect-error sourcemap exists + map: ref.sourcemap as ExistingRawSourceMap | undefined, + deps: ref.deps(), + } + } + }, + { + shouldUseFake(_stylusPath, _content, _root, options) { + return Object.values(options.define).some( + (d) => typeof d === 'function', + ) + }, + }, + ) + return worker +} - // Concat imports deps with computed deps - const deps = [...ref.deps(), ...importsDeps] +const stylProcessor = (): StylusStylePreprocessor => { + const workerMap = new Map>() - // @ts-expect-error sourcemap exists - const map: ExistingRawSourceMap | undefined = ref.sourcemap + return { + close() { + for (const worker of workerMap.values()) { + worker.stop() + } + }, + async process(source, root, options, resolvers) { + const stylusPath = loadPreprocessorPath(PreprocessLang.stylus, root) - return { - code: result, - map: formatStylusSourceMap(map, root), - additionalMap, - deps, - } - } catch (e) { - e.message = `[stylus] ${e.message}` - return { code: '', error: e, deps: [] } + if (!workerMap.has(options.alias)) { + workerMap.set(options.alias, makeStylWorker()) + } + const worker = workerMap.get(options.alias)! + + // Get source with preprocessor options.additionalData. Make sure a new line separator + // is added to avoid any render error, as added stylus content may not have semi-colon separators + const { content, map: additionalMap } = await getSource( + source, + options.filename, + options.additionalData, + options.enableSourcemap, + '\n', + ) + // Get preprocessor options.imports dependencies as stylus + // does not return them with its builtin `.deps()` method + const importsDeps = (options.imports ?? []).map((dep: string) => + path.resolve(dep), + ) + try { + const { code, map, deps } = await worker.run( + stylusPath, + content, + root, + options, + ) + return { + code, + map: formatStylusSourceMap(map, root), + additionalMap, + // Concat imports deps with computed deps + deps: [...deps, ...importsDeps], + } + } catch (e) { + const wrapped = new Error(`[stylus] ${e.message}`, { cause: e }) + return { code: '', error: wrapped, deps: [] } + } + }, } } @@ -2213,16 +2321,50 @@ async function getSource( } } -const preProcessors = Object.freeze({ - [PreprocessLang.less]: less, - [PreprocessLang.sass]: sass, - [PreprocessLang.scss]: scss, - [PreprocessLang.styl]: styl, - [PreprocessLang.stylus]: styl, -}) +const createPreprocessorWorkerController = () => { + const scss = scssProcessor() + const less = lessProcessor() + const styl = stylProcessor() + + const sassProcess: StylePreprocessor['process'] = ( + source, + root, + options, + resolvers, + ) => { + return scss.process(source, root, options, resolvers) + } + + const close = () => { + less.close() + scss.close() + styl.close() + } + + return { + [PreprocessLang.less]: less.process, + [PreprocessLang.scss]: scss.process, + [PreprocessLang.sass]: sassProcess, + [PreprocessLang.styl]: styl.process, + [PreprocessLang.stylus]: styl.process, + close, + } as const satisfies { [K in PreprocessLang | 'close']: unknown } +} + +type PreprocessorWorkerController = ReturnType< + typeof createPreprocessorWorkerController +> + +const preprocessorSet = new Set([ + PreprocessLang.less, + PreprocessLang.sass, + PreprocessLang.scss, + PreprocessLang.styl, + PreprocessLang.stylus, +] as const) function isPreProcessor(lang: any): lang is PreprocessLang { - return lang && lang in preProcessors + return lang && preprocessorSet.has(lang) } const importLightningCSS = createCachedImport(() => import('lightningcss')) diff --git a/packages/vite/src/node/plugins/terser.ts b/packages/vite/src/node/plugins/terser.ts index 03f12ff464e968..cf73817af665b0 100644 --- a/packages/vite/src/node/plugins/terser.ts +++ b/packages/vite/src/node/plugins/terser.ts @@ -26,16 +26,17 @@ const loadTerserPath = (root: string) => { export function terserPlugin(config: ResolvedConfig): Plugin { const makeWorker = () => new Worker( - async ( - terserPath: string, - code: string, - options: Terser.MinifyOptions, - ) => { - // test fails when using `import`. maybe related: https://github.com/nodejs/node/issues/43205 - // eslint-disable-next-line no-restricted-globals -- this function runs inside cjs - const terser = require(terserPath) - return terser.minify(code, options) as Terser.MinifyOutput - }, + () => + async ( + terserPath: string, + code: string, + options: Terser.MinifyOptions, + ) => { + // test fails when using `import`. maybe related: https://github.com/nodejs/node/issues/43205 + // eslint-disable-next-line no-restricted-globals -- this function runs inside cjs + const terser = require(terserPath) + return terser.minify(code, options) as Terser.MinifyOutput + }, ) let worker: ReturnType From e8f4fea2a11a248cbfcf9cb7e9f448319717812c Mon Sep 17 00:00:00 2001 From: sapphi-red Date: Sun, 13 Aug 2023 23:56:22 +0900 Subject: [PATCH 03/16] wip: fix misc --- packages/vite/src/node/plugins/css.ts | 54 +++++++++++++++++++++------ 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index c51b2a1c7aceb9..7a2491a3ae5327 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -1824,7 +1824,8 @@ const makeScssWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { async ( sassPath: string, data: string, - options: SassStylePreprocessorOptions, + // additionalData can a function that is not cloneable but it won't be used + options: SassStylePreprocessorOptions & { additionalData: undefined }, ) => { // eslint-disable-next-line no-restricted-globals const sass: typeof Sass = require(sassPath) @@ -1915,8 +1916,16 @@ const scssProcessor = (): SassStylePreprocessor => { options.enableSourcemap, ) + const optionsWithoutAdditionalData = { + ...options, + additionalData: undefined, + } try { - const result = await worker.run(sassPath, data, options) + const result = await worker.run( + sassPath, + data, + optionsWithoutAdditionalData, + ) const deps = result.stats.includedFiles.map((f) => cleanScssBugUrl(f)) const map: ExistingRawSourceMap | undefined = result.map ? JSON.parse(result.map.toString()) @@ -2038,7 +2047,8 @@ const makeLessWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { less: typeof Less, rootFile: string, ): Less.Plugin => { - ViteLessManager ??= class ViteManager extends less.FileManager { + const { FileManager } = less + ViteLessManager ??= class ViteManager extends FileManager { rootFile constructor(rootFile: string) { super() @@ -2081,7 +2091,8 @@ const makeLessWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { return async ( lessPath: string, content: string, - options: StylePreprocessorOptions, + // additionalData can a function that is not cloneable but it won't be used + options: StylePreprocessorOptions & { additionalData: undefined }, ) => { // eslint-disable-next-line no-restricted-globals const nodeLess: typeof Less = require(lessPath) @@ -2139,8 +2150,16 @@ const lessProcessor = (): StylePreprocessor => { ) let result: Less.RenderOutput | undefined + const optionsWithoutAdditionalData = { + ...options, + additionalData: undefined, + } try { - result = await worker.run(lessPath, content, options) + result = await worker.run( + lessPath, + content, + optionsWithoutAdditionalData, + ) } catch (e) { const error = e as Less.RenderError // normalize error info @@ -2178,7 +2197,8 @@ const makeStylWorker = () => { stylusPath: string, content: string, root: string, - options: StylePreprocessorOptions, + // additionalData can a function that is not cloneable but it won't be used + options: StylusStylePreprocessorOptions & { additionalData: undefined }, ) => { // eslint-disable-next-line no-restricted-globals const nodeStylus: typeof Stylus = require(stylusPath) @@ -2207,8 +2227,9 @@ const makeStylWorker = () => { }, { shouldUseFake(_stylusPath, _content, _root, options) { - return Object.values(options.define).some( - (d) => typeof d === 'function', + return !!( + options.define && + Object.values(options.define).some((d) => typeof d === 'function') ) }, }, @@ -2247,12 +2268,16 @@ const stylProcessor = (): StylusStylePreprocessor => { const importsDeps = (options.imports ?? []).map((dep: string) => path.resolve(dep), ) + const optionsWithoutAdditionalData = { + ...options, + additionalData: undefined, + } try { const { code, map, deps } = await worker.run( stylusPath, content, root, - options, + optionsWithoutAdditionalData, ) return { code, @@ -2262,7 +2287,9 @@ const stylProcessor = (): StylusStylePreprocessor => { deps: [...deps, ...importsDeps], } } catch (e) { - const wrapped = new Error(`[stylus] ${e.message}`, { cause: e }) + const wrapped = new Error(`[stylus] ${e.message}`) + wrapped.name = e.name + wrapped.stack = e.stack return { code: '', error: wrapped, deps: [] } } }, @@ -2332,7 +2359,12 @@ const createPreprocessorWorkerController = () => { options, resolvers, ) => { - return scss.process(source, root, options, resolvers) + return scss.process( + source, + root, + { ...options, indentedSyntax: true }, + resolvers, + ) } const close = () => { From da28fdc372bf26e464cd475a6326ff625d4aeeea Mon Sep 17 00:00:00 2001 From: sapphi-red Date: Tue, 15 Aug 2023 20:59:20 +0900 Subject: [PATCH 04/16] fix: correctly close preprocessor workers --- packages/vite/src/node/plugins/css.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 7a2491a3ae5327..8965a98375a112 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -1267,7 +1267,7 @@ export async function preprocessCSS( ): Promise { const p = createCSSPreprocessor() try { - return p.process(code, filename, config) + return await p.process(code, filename, config) } finally { p.close() } From b9ed29aed5b208818da4c8cd00aad7afeb5150fa Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Wed, 8 Nov 2023 01:48:07 +0900 Subject: [PATCH 05/16] refactor: use artichokie --- packages/vite/LICENSE.md | 30 +++ packages/vite/package.json | 2 +- packages/vite/src/node/okie.ts | 255 ----------------------- packages/vite/src/node/plugins/css.ts | 2 +- packages/vite/src/node/plugins/terser.ts | 2 +- pnpm-lock.yaml | 16 +- 6 files changed, 41 insertions(+), 266 deletions(-) delete mode 100644 packages/vite/src/node/okie.ts diff --git a/packages/vite/LICENSE.md b/packages/vite/LICENSE.md index 5197a318bb3260..acaed4bfdbd8bb 100644 --- a/packages/vite/LICENSE.md +++ b/packages/vite/LICENSE.md @@ -657,6 +657,36 @@ Repository: https://github.com/micromatch/anymatch --------------------------------------- +## artichokie +License: MIT +By: sapphi-red, Evan You +Repository: git+https://github.com/sapphi-red/artichokie.git + +> MIT License +> +> Copyright (c) 2020-present, Yuxi (Evan) You +> Copyright (c) 2023-present, sapphi-red +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. + +--------------------------------------- + ## astring License: MIT By: David Bonnet diff --git a/packages/vite/package.json b/packages/vite/package.json index 26e611eedbab23..194e038affa000 100644 --- a/packages/vite/package.json +++ b/packages/vite/package.json @@ -96,6 +96,7 @@ "@types/pnpapi": "^0.0.4", "acorn": "^8.11.2", "acorn-walk": "^8.3.0", + "artichokie": "^0.1.0", "cac": "^6.7.14", "chokidar": "^3.5.3", "connect": "^3.7.0", @@ -119,7 +120,6 @@ "micromatch": "^4.0.5", "mlly": "^1.4.2", "mrmime": "^1.0.1", - "okie": "^1.0.1", "open": "^8.4.2", "parse5": "^7.1.2", "periscopic": "^4.0.2", diff --git a/packages/vite/src/node/okie.ts b/packages/vite/src/node/okie.ts deleted file mode 100644 index 0125f4656ff9d0..00000000000000 --- a/packages/vite/src/node/okie.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { createRequire } from 'node:module' -import os from 'node:os' -import { Worker as _Worker } from 'node:worker_threads' - -interface NodeWorker extends _Worker { - currentResolve: ((value: any) => void) | null - currentReject: ((err: Error) => void) | null -} - -export interface Options { - max?: number - parentFunctions?: Record Promise> -} - -export class Worker { - private code: string - private parentFunctions: Record Promise> - private max: number - private pool: NodeWorker[] - private idlePool: NodeWorker[] - private queue: [(worker: NodeWorker) => void, (err: Error) => void][] - - constructor( - fn: () => (...args: Args) => Promise | Ret, - options: Options = {}, - ) { - this.code = genWorkerCode(fn, options.parentFunctions ?? {}) - this.parentFunctions = options.parentFunctions ?? {} - this.max = options.max || Math.max(1, os.cpus().length - 1) - this.pool = [] - this.idlePool = [] - this.queue = [] - } - - async run(...args: Args): Promise { - const worker = await this._getAvailableWorker() - return new Promise((resolve, reject) => { - worker.currentResolve = resolve - worker.currentReject = reject - worker.postMessage({ type: 'run', args }) - }) - } - - stop(): void { - this.pool.forEach((w) => w.unref()) - this.queue.forEach(([_, reject]) => - reject( - new Error('Main worker pool stopped before a worker was available.'), - ), - ) - this.pool = [] - this.idlePool = [] - this.queue = [] - } - - private async _getAvailableWorker(): Promise { - // has idle one? - if (this.idlePool.length) { - return this.idlePool.shift()! - } - - // can spawn more? - if (this.pool.length < this.max) { - const worker = new _Worker(this.code, { eval: true }) as NodeWorker - - worker.on('message', async (args) => { - if (args.type === 'run') { - if ('result' in args) { - worker.currentResolve && worker.currentResolve(args.result) - worker.currentResolve = null - this._assignDoneWorker(worker) - } else { - worker.currentReject && worker.currentReject(args.error) - worker.currentReject = null - } - } else if (args.type === 'parentFunction') { - if (!(args.name in this.parentFunctions)) { - throw new Error( - `Parent function ${JSON.stringify( - args.name, - )} was not passed to options but was called.`, - ) - } - - try { - const result = await this.parentFunctions[args.name](...args.args) - worker.postMessage({ type: 'parentFunction', id: args.id, result }) - } catch (e) { - worker.postMessage({ - type: 'parentFunction', - id: args.id, - error: e, - }) - } - } - }) - - worker.on('error', (err) => { - worker.currentReject && worker.currentReject(err) - worker.currentReject = null - }) - - worker.on('exit', (code) => { - const i = this.pool.indexOf(worker) - if (i > -1) this.pool.splice(i, 1) - if (code !== 0 && worker.currentReject) { - worker.currentReject( - new Error(`Worker stopped with non-0 exit code ${code}`), - ) - worker.currentReject = null - } - }) - - this.pool.push(worker) - return worker - } - - // no one is available, we have to wait - let resolve: (worker: NodeWorker) => void - let reject: (err: Error) => any - const onWorkerAvailablePromise = new Promise((r, rj) => { - resolve = r - reject = rj - }) - this.queue.push([resolve!, reject!]) - return onWorkerAvailablePromise - } - - private _assignDoneWorker(worker: NodeWorker) { - // someone's waiting already? - if (this.queue.length) { - const [resolve] = this.queue.shift()! - resolve(worker) - return - } - // take a rest. - this.idlePool.push(worker) - } -} - -function genWorkerCode(fn: Function, parentFunctions: Record) { - return ` -let id = 0 -const parentFunctionResolvers = new Map() -const parentFunctionCall = (key) => async (...args) => { - id++ - let resolve, reject - const promise = new Promise((res, rej) => { - resolve = res - reject = rej - }) - parentFunctionResolvers.set(id, { resolve, reject }) - - parentPort.postMessage({ type: 'parentFunction', id, name: key, args }) - return await promise -} - -const doWork = (() => { - ${Object.keys(parentFunctions) - .map((key) => `const ${key} = parentFunctionCall(${JSON.stringify(key)});`) - .join('\n')} - return (${fn.toString()})() -})() - -const { parentPort } = require('worker_threads') - -parentPort.on('message', async (args) => { - if (args.type === 'run') { - try { - const res = await doWork(...args.args) - parentPort.postMessage({ type: 'run', result: res }) - } catch (e) { - parentPort.postMessage({ type: 'run', error: e }) - } - } else if (args.type === 'parentFunction') { - const id = args.id - if (parentFunctionResolvers.has(id)) { - const { resolve, reject } = parentFunctionResolvers.get(id) - parentFunctionResolvers.delete(id) - - if ('result' in args) { - resolve(args.result) - } else { - reject(args.error) - } - } - } -}) - ` -} - -class FakeWorker { - private fn: (...args: Args) => Promise - - constructor( - fn: () => (...args: Args) => Promise | Ret, - options: Options = {}, - ) { - const argsAndCode = genFakeWorkerArgsAndCode( - fn, - options.parentFunctions ?? {}, - ) - const require = createRequire(import.meta.url) - this.fn = new Function(...argsAndCode)(require, options.parentFunctions) - } - - async run(...args: Args): Promise { - return this.fn(...args) - } - - stop(): void { - /* no-op */ - } -} - -function genFakeWorkerArgsAndCode( - fn: Function, - parentFunctions: Record, -) { - return [ - 'require', - 'parentFunctions', - ` -${Object.keys(parentFunctions) - .map((key) => `const ${key} = parentFunctions[${JSON.stringify(key)}];`) - .join('\n')} -return (${fn.toString()})() - `, - ] -} - -export class WorkerWithFallback { - private _realWorker: Worker - private _fakeWorker: FakeWorker - private _shouldUseFake: (...args: Args) => boolean - - constructor( - fn: () => (...args: Args) => Promise | Ret, - options: Options & { shouldUseFake: (...args: Args) => boolean }, - ) { - this._realWorker = new Worker(fn, options) - this._fakeWorker = new FakeWorker(fn, options) - this._shouldUseFake = options.shouldUseFake - } - - async run(...args: Args): Promise { - const useFake = this._shouldUseFake(...args) - return this[useFake ? '_fakeWorker' : '_realWorker'].run(...args) - } - - stop(): void { - this._realWorker.stop() - this._fakeWorker.stop() - } -} diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 1185ef8a93b2d0..8a370e4c7b626a 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -24,6 +24,7 @@ import type { LightningCSSOptions } from 'dep-types/lightningcss' import type { TransformOptions } from 'esbuild' import { formatMessages, transform } from 'esbuild' import type { RawSourceMap } from '@ampproject/remapping' +import { WorkerWithFallback } from 'artichokie' import { getCodeWithSourcemap, injectSourcesContent } from '../server/sourcemap' import type { ModuleNode } from '../server/moduleGraph' import type { ResolveFn, ViteDevServer } from '../' @@ -58,7 +59,6 @@ import { stripBomTag, } from '../utils' import type { Logger } from '../logger' -import { WorkerWithFallback } from '../okie' import { addToHTMLProxyTransformResult } from './html' import { assetUrlRE, diff --git a/packages/vite/src/node/plugins/terser.ts b/packages/vite/src/node/plugins/terser.ts index 3029667540ab74..90c29b26c7501e 100644 --- a/packages/vite/src/node/plugins/terser.ts +++ b/packages/vite/src/node/plugins/terser.ts @@ -1,5 +1,5 @@ import type { Terser } from 'dep-types/terser' -import { Worker } from '../okie' +import { Worker } from 'artichokie' import type { Plugin } from '../plugin' import type { ResolvedConfig } from '..' import { requireResolveFromRootWithFallback } from '../utils' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 907924494bdd86..4b547d2345e131 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -279,6 +279,9 @@ importers: acorn-walk: specifier: ^8.3.0 version: 8.3.0(acorn@8.11.2) + artichokie: + specifier: ^0.1.0 + version: 0.1.0 cac: specifier: ^6.7.14 version: 6.7.14 @@ -348,9 +351,6 @@ importers: mrmime: specifier: ^1.0.1 version: 1.0.1 - okie: - specifier: ^1.0.1 - version: 1.0.1 open: specifier: ^8.4.2 version: 8.4.2 @@ -4581,6 +4581,11 @@ packages: engines: {node: '>=0.10.0'} dev: true + /artichokie@0.1.0: + resolution: {integrity: sha512-a1aMGGXwENT6PKlAcMGuc18mGIf+BoZ5zecvuJxVga6UTjiiMq8LSaT9FIni06YmBLIhP4l1u9jES4uZ97WE9w==} + engines: {node: ^18.0.0 || >=20.0.0} + dev: true + /as-table@1.0.55: resolution: {integrity: sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==} dependencies: @@ -7827,11 +7832,6 @@ packages: es-abstract: 1.22.1 dev: true - /okie@1.0.1: - resolution: {integrity: sha512-JQh5TdSYhzXSuKN3zzX8Rw9Q/Tec1fm0jwP/k9+cBDk6tyLjlARVu936MLY//2NZp76UGHH+5gXPzRejU1bTjQ==} - engines: {node: '>=12.0.0'} - dev: true - /on-finished@2.3.0: resolution: {integrity: sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==} engines: {node: '>= 0.8'} From 78edaf7f94ef7e283e400ce34236a4f37c55ebb2 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 1 Dec 2023 19:32:53 +0900 Subject: [PATCH 06/16] docs: split additionalData option --- docs/config/shared-options.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/config/shared-options.md b/docs/config/shared-options.md index e6ac34f36e2ef5..8d4e0ba62d86d7 100644 --- a/docs/config/shared-options.md +++ b/docs/config/shared-options.md @@ -222,17 +222,12 @@ Specify options to pass to CSS pre-processors. The file extensions are used as k - `less` - [Options](https://lesscss.org/usage/#less-options). - `styl`/`stylus` - Only [`define`](https://stylus-lang.com/docs/js.html#define-name-node) is supported, which can be passed as an object. -All preprocessor options also support the `additionalData` option, which can be used to inject extra code for each style content. Note that if you include actual styles and not just variables, those styles will be duplicated in the final bundle. - -Example: +**Example:** ```js export default defineConfig({ css: { preprocessorOptions: { - scss: { - additionalData: `$injectedColor: orange;`, - }, less: { math: 'parens-division', }, @@ -246,6 +241,26 @@ export default defineConfig({ }) ``` +### css.preprocessorOptions[extension].additionalData + +- **Type:** `string | ((source: string, filename: string) => (string | { content: string; map?: SourceMap }))` + +This option can be used to inject extra code for each style content. Note that if you include actual styles and not just variables, those styles will be duplicated in the final bundle. + +**Example:** + +```js +export default defineConfig({ + css: { + preprocessorOptions: { + scss: { + additionalData: `$injectedColor: orange;`, + }, + }, + }, +}) +``` + ## css.devSourcemap - **Experimental:** [Give Feedback](https://github.com/vitejs/vite/discussions/13845) From b9fa5893c8410a5048b66c9f8238b3936fabf6b9 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 1 Dec 2023 19:35:10 +0900 Subject: [PATCH 07/16] feat: add `preprocessorOptions[ext].maxWorkers` --- packages/vite/package.json | 2 +- packages/vite/src/node/plugins/css.ts | 36 ++++++++++++++++++++++----- playground/css/vite.config.js | 5 ++++ pnpm-lock.yaml | 8 +++--- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/packages/vite/package.json b/packages/vite/package.json index 0750394f5e13ab..8bb1235062fd17 100644 --- a/packages/vite/package.json +++ b/packages/vite/package.json @@ -96,7 +96,7 @@ "@types/pnpapi": "^0.0.5", "acorn": "^8.11.2", "acorn-walk": "^8.3.0", - "artichokie": "^0.1.0", + "artichokie": "^0.2.0", "cac": "^6.7.14", "chokidar": "^3.5.3", "connect": "^3.7.0", diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 1694f7877a5c91..bf989dbe72f705 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -1756,6 +1756,7 @@ type PreprocessorAdditionalData = type StylePreprocessorOptions = { [key: string]: any additionalData?: PreprocessorAdditionalData + maxWorkers?: number | true filename: string alias: Alias[] enableSourcemap: boolean @@ -1884,7 +1885,11 @@ function fixScssBugImportValue( } // .scss/.sass processor -const makeScssWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { +const makeScssWorker = ( + resolvers: CSSAtImportResolvers, + alias: Alias[], + maxWorkers: number | true | undefined, +) => { const internalImporter = async ( url: string, importer: string, @@ -1972,6 +1977,7 @@ const makeScssWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { (!Array.isArray(options.importer) || options.importer.length > 0)) ) }, + max: normalizeMaxWorkers(maxWorkers), }, ) return worker @@ -1990,7 +1996,10 @@ const scssProcessor = (): SassStylePreprocessor => { const sassPath = loadPreprocessorPath(PreprocessLang.sass, root) if (!workerMap.has(options.alias)) { - workerMap.set(options.alias, makeScssWorker(resolvers, options.alias)) + workerMap.set( + options.alias, + makeScssWorker(resolvers, options.alias, options.maxWorkers), + ) } const worker = workerMap.get(options.alias)! @@ -2101,7 +2110,11 @@ async function rebaseUrls( } // .less -const makeLessWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { +const makeLessWorker = ( + resolvers: CSSAtImportResolvers, + alias: Alias[], + maxWorkers: number | true | undefined, +) => { const viteLessResolve = async ( filename: string, dir: string, @@ -2205,6 +2218,7 @@ const makeLessWorker = (resolvers: CSSAtImportResolvers, alias: Alias[]) => { shouldUseFake(_lessPath, _content, options) { return options.plugins?.length > 0 }, + max: normalizeMaxWorkers(maxWorkers), }, ) return worker @@ -2223,7 +2237,10 @@ const lessProcessor = (): StylePreprocessor => { const lessPath = loadPreprocessorPath(PreprocessLang.less, root) if (!workerMap.has(options.alias)) { - workerMap.set(options.alias, makeLessWorker(resolvers, options.alias)) + workerMap.set( + options.alias, + makeLessWorker(resolvers, options.alias, options.maxWorkers), + ) } const worker = workerMap.get(options.alias)! @@ -2275,7 +2292,7 @@ const lessProcessor = (): StylePreprocessor => { } // .styl -const makeStylWorker = () => { +const makeStylWorker = (maxWorkers: number | true | undefined) => { const worker = new WorkerWithFallback( () => { return async ( @@ -2317,6 +2334,7 @@ const makeStylWorker = () => { Object.values(options.define).some((d) => typeof d === 'function') ) }, + max: normalizeMaxWorkers(maxWorkers), }, ) return worker @@ -2335,7 +2353,7 @@ const stylProcessor = (): StylusStylePreprocessor => { const stylusPath = loadPreprocessorPath(PreprocessLang.stylus, root) if (!workerMap.has(options.alias)) { - workerMap.set(options.alias, makeStylWorker()) + workerMap.set(options.alias, makeStylWorker(options.maxWorkers)) } const worker = workerMap.get(options.alias)! @@ -2468,6 +2486,12 @@ const createPreprocessorWorkerController = () => { } as const satisfies { [K in PreprocessLang | 'close']: unknown } } +const normalizeMaxWorkers = (maxWorker: number | true | undefined) => { + if (maxWorker === undefined) return 0 + if (maxWorker === true) return undefined + return maxWorker +} + type PreprocessorWorkerController = ReturnType< typeof createPreprocessorWorkerController > diff --git a/playground/css/vite.config.js b/playground/css/vite.config.js index 3d301ae03bec3e..2beea8b6846b1f 100644 --- a/playground/css/vite.config.js +++ b/playground/css/vite.config.js @@ -59,6 +59,9 @@ export default defineConfig({ // }, }, preprocessorOptions: { + less: { + maxWorkers: true, + }, scss: { additionalData: `$injectedColor: orange;`, importer: [ @@ -69,6 +72,7 @@ export default defineConfig({ return url.endsWith('.wxss') ? { contents: '' } : null }, ], + maxWorkers: true, }, styl: { additionalData: `$injectedColor ?= orange`, @@ -80,6 +84,7 @@ export default defineConfig({ $definedColor: new stylus.nodes.RGBA(51, 197, 255, 1), definedFunction: () => new stylus.nodes.RGBA(255, 0, 98, 1), }, + maxWorkers: true, }, }, }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12618a4ed132c8..7205d5afda1f18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -292,8 +292,8 @@ importers: specifier: ^8.3.0 version: 8.3.0(acorn@8.11.2) artichokie: - specifier: ^0.1.0 - version: 0.1.0 + specifier: ^0.2.0 + version: 0.2.0 cac: specifier: ^6.7.14 version: 6.7.14 @@ -4639,8 +4639,8 @@ packages: is-shared-array-buffer: 1.0.2 dev: true - /artichokie@0.1.0: - resolution: {integrity: sha512-a1aMGGXwENT6PKlAcMGuc18mGIf+BoZ5zecvuJxVga6UTjiiMq8LSaT9FIni06YmBLIhP4l1u9jES4uZ97WE9w==} + /artichokie@0.2.0: + resolution: {integrity: sha512-LXtOFWUNABHEo49FJpwOf8VLzOJ1iGV9xu9ezwnveI75LIqGhUDDjMFo3MkUmtc+t3oDZRMATuVMrt6d8FCvrQ==} engines: {node: ^18.0.0 || >=20.0.0} dev: true From 3d7c3142526646c5586a27092fa8641af0a8a842 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 1 Dec 2023 19:36:02 +0900 Subject: [PATCH 08/16] docs: add `preprocessorOptions[ext].maxWorkers` --- docs/config/shared-options.md | 8 ++++++++ packages/vite/src/node/plugins/css.ts | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/docs/config/shared-options.md b/docs/config/shared-options.md index 8d4e0ba62d86d7..9f91558afb2b1e 100644 --- a/docs/config/shared-options.md +++ b/docs/config/shared-options.md @@ -261,6 +261,14 @@ export default defineConfig({ }) ``` +### css.preprocessorOptions[extension].maxWorkers + +- **Experimental:** [Give Feedback](https://github.com/vitejs/vite/discussions/13845) +- **Type:** `number | true` +- **Default:** `0` (does not create any workers and run in the main thread) + +If this option is set, preprocessors will run in workers when possible. `true` means the number of CPUs minus 1. + ## css.devSourcemap - **Experimental:** [Give Feedback](https://github.com/vitejs/vite/discussions/13845) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index bf989dbe72f705..00a5d7471536c8 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -90,6 +90,16 @@ export interface CSSOptions { * https://github.com/css-modules/postcss-modules */ modules?: CSSModulesOptions | false + /** + * Options for preprocessors. + * + * In addition to options specific to each processors, + * Vite supports `additionalData` option and `maxWorkers` option. + * + * The `additionalData` option can be used to inject extra code for each style content. + * + * The experimental `maxWorkers: number | true` option specify the max number of workers to spawn. + */ preprocessorOptions?: Record postcss?: | string From 71c8dd2128aaec98aebd294e4e96192c6fe1a6a2 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Sun, 3 Dec 2023 00:54:45 +0900 Subject: [PATCH 09/16] chore: rename to `css.preprocessorMaxWorkers` --- docs/config/shared-options.md | 4 +-- packages/vite/src/node/plugins/css.ts | 44 ++++++++++++++++----------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/docs/config/shared-options.md b/docs/config/shared-options.md index 9f91558afb2b1e..4bdebf84b0eee8 100644 --- a/docs/config/shared-options.md +++ b/docs/config/shared-options.md @@ -261,9 +261,9 @@ export default defineConfig({ }) ``` -### css.preprocessorOptions[extension].maxWorkers +## css.preprocessorMaxWorkers -- **Experimental:** [Give Feedback](https://github.com/vitejs/vite/discussions/13845) +- **Experimental:** [Give Feedback](TODO: update) - **Type:** `number | true` - **Default:** `0` (does not create any workers and run in the main thread) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 00a5d7471536c8..33bf795b2294f8 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -93,14 +93,18 @@ export interface CSSOptions { /** * Options for preprocessors. * - * In addition to options specific to each processors, - * Vite supports `additionalData` option and `maxWorkers` option. - * + * In addition to options specific to each processors, Vite supports `additionalData` option. * The `additionalData` option can be used to inject extra code for each style content. - * - * The experimental `maxWorkers: number | true` option specify the max number of workers to spawn. */ preprocessorOptions?: Record + /** + * If this option is set, preprocessors will run in workers when possible. + * `true` means the number of CPUs minus 1. + * + * @default 0 + * @experimental + */ + preprocessorMaxWorkers?: number | true postcss?: | string | (PostCSS.ProcessOptions & { @@ -979,6 +983,7 @@ async function compileCSSPreprocessors( config.root, opts, atImportResolvers, + normalizeMaxWorkers(config.css.preprocessorMaxWorkers), ) if (preprocessResult.error) { throw preprocessResult.error @@ -1784,6 +1789,7 @@ type StylePreprocessor = { root: string, options: StylePreprocessorOptions, resolvers: CSSAtImportResolvers, + maxWorkers: number | undefined, ) => StylePreprocessorResults | Promise close: () => void } @@ -1794,6 +1800,7 @@ type SassStylePreprocessor = { root: string, options: SassStylePreprocessorOptions, resolvers: CSSAtImportResolvers, + maxWorkers: number | undefined, ) => StylePreprocessorResults | Promise close: () => void } @@ -1804,6 +1811,7 @@ type StylusStylePreprocessor = { root: string, options: StylusStylePreprocessorOptions, resolvers: CSSAtImportResolvers, + maxWorkers: number | undefined, ) => StylePreprocessorResults | Promise close: () => void } @@ -1898,7 +1906,7 @@ function fixScssBugImportValue( const makeScssWorker = ( resolvers: CSSAtImportResolvers, alias: Alias[], - maxWorkers: number | true | undefined, + maxWorkers: number | undefined, ) => { const internalImporter = async ( url: string, @@ -1987,7 +1995,7 @@ const makeScssWorker = ( (!Array.isArray(options.importer) || options.importer.length > 0)) ) }, - max: normalizeMaxWorkers(maxWorkers), + max: maxWorkers, }, ) return worker @@ -2002,13 +2010,13 @@ const scssProcessor = (): SassStylePreprocessor => { worker.stop() } }, - async process(source, root, options, resolvers) { + async process(source, root, options, resolvers, maxWorkers) { const sassPath = loadPreprocessorPath(PreprocessLang.sass, root) if (!workerMap.has(options.alias)) { workerMap.set( options.alias, - makeScssWorker(resolvers, options.alias, options.maxWorkers), + makeScssWorker(resolvers, options.alias, maxWorkers), ) } const worker = workerMap.get(options.alias)! @@ -2123,7 +2131,7 @@ async function rebaseUrls( const makeLessWorker = ( resolvers: CSSAtImportResolvers, alias: Alias[], - maxWorkers: number | true | undefined, + maxWorkers: number | undefined, ) => { const viteLessResolve = async ( filename: string, @@ -2228,7 +2236,7 @@ const makeLessWorker = ( shouldUseFake(_lessPath, _content, options) { return options.plugins?.length > 0 }, - max: normalizeMaxWorkers(maxWorkers), + max: maxWorkers, }, ) return worker @@ -2243,13 +2251,13 @@ const lessProcessor = (): StylePreprocessor => { worker.stop() } }, - async process(source, root, options, resolvers) { + async process(source, root, options, resolvers, maxWorkers) { const lessPath = loadPreprocessorPath(PreprocessLang.less, root) if (!workerMap.has(options.alias)) { workerMap.set( options.alias, - makeLessWorker(resolvers, options.alias, options.maxWorkers), + makeLessWorker(resolvers, options.alias, maxWorkers), ) } const worker = workerMap.get(options.alias)! @@ -2302,7 +2310,7 @@ const lessProcessor = (): StylePreprocessor => { } // .styl -const makeStylWorker = (maxWorkers: number | true | undefined) => { +const makeStylWorker = (maxWorkers: number | undefined) => { const worker = new WorkerWithFallback( () => { return async ( @@ -2344,7 +2352,7 @@ const makeStylWorker = (maxWorkers: number | true | undefined) => { Object.values(options.define).some((d) => typeof d === 'function') ) }, - max: normalizeMaxWorkers(maxWorkers), + max: maxWorkers, }, ) return worker @@ -2359,11 +2367,11 @@ const stylProcessor = (): StylusStylePreprocessor => { worker.stop() } }, - async process(source, root, options, resolvers) { + async process(source, root, options, resolvers, maxWorkers) { const stylusPath = loadPreprocessorPath(PreprocessLang.stylus, root) if (!workerMap.has(options.alias)) { - workerMap.set(options.alias, makeStylWorker(options.maxWorkers)) + workerMap.set(options.alias, makeStylWorker(maxWorkers)) } const worker = workerMap.get(options.alias)! @@ -2471,12 +2479,14 @@ const createPreprocessorWorkerController = () => { root, options, resolvers, + maxWorkers, ) => { return scss.process( source, root, { ...options, indentedSyntax: true }, resolvers, + maxWorkers, ) } From ee56ac47930f6d8760d191a5ca77c2a72f6e6f02 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 19 Jan 2024 01:16:00 +0900 Subject: [PATCH 10/16] test: use preprocessorMaxWorkers --- playground/css/vite.config.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/playground/css/vite.config.js b/playground/css/vite.config.js index 2beea8b6846b1f..5ac9d448a2734a 100644 --- a/playground/css/vite.config.js +++ b/playground/css/vite.config.js @@ -59,9 +59,6 @@ export default defineConfig({ // }, }, preprocessorOptions: { - less: { - maxWorkers: true, - }, scss: { additionalData: `$injectedColor: orange;`, importer: [ @@ -72,7 +69,6 @@ export default defineConfig({ return url.endsWith('.wxss') ? { contents: '' } : null }, ], - maxWorkers: true, }, styl: { additionalData: `$injectedColor ?= orange`, @@ -84,8 +80,8 @@ export default defineConfig({ $definedColor: new stylus.nodes.RGBA(51, 197, 255, 1), definedFunction: () => new stylus.nodes.RGBA(255, 0, 98, 1), }, - maxWorkers: true, }, }, + preprocessorMaxWorkers: true, }, }) From ffe13c6bf2fbd8c0f8bdba90f34e86a114aafd26 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 19 Jan 2024 01:17:00 +0900 Subject: [PATCH 11/16] fix: call `createPreprocessorWorkerController` in buildStart --- packages/vite/src/node/plugins/css.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 3d09c57ff90339..1db33c84da0c30 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -261,7 +261,7 @@ export function cssPlugin(config: ResolvedConfig): Plugin { extensions: [], }) - const preprocessorWorkerController = createPreprocessorWorkerController() + let preprocessorWorkerController: PreprocessorWorkerController | undefined // warm up cache for resolved postcss config if (config.css?.transformer !== 'lightningcss') { @@ -281,6 +281,12 @@ export function cssPlugin(config: ResolvedConfig): Plugin { cssModulesCache.set(config, moduleCache) removedPureCssFilesCache.set(config, new Map()) + + preprocessorWorkerController = createPreprocessorWorkerController() + }, + + buildEnd() { + preprocessorWorkerController?.close() }, async load(id) { @@ -361,7 +367,7 @@ export function cssPlugin(config: ResolvedConfig): Plugin { id, raw, config, - preprocessorWorkerController, + preprocessorWorkerController!, urlReplacer, ) if (modules) { @@ -427,9 +433,6 @@ export function cssPlugin(config: ResolvedConfig): Plugin { map, } }, - buildEnd() { - preprocessorWorkerController.close() - }, } } From f493c137ca26e7a4851596d4476defc65da386a2 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 19 Jan 2024 01:18:01 +0900 Subject: [PATCH 12/16] refactor: move `getPackageManagerCommand('install')` --- packages/vite/src/node/plugins/css.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 1db33c84da0c30..f73087e969d2b1 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -1964,8 +1964,8 @@ function loadPreprocessorPath( const resolved = requireResolveFromRootWithFallback(root, lang) return (loadedPreprocessorPath[lang] = resolved) } catch (e) { - const installCommand = getPackageManagerCommand('install') if (e.code === 'MODULE_NOT_FOUND') { + const installCommand = getPackageManagerCommand('install') throw new Error( `Preprocessor dependency "${lang}" not found. Did you install it? Try \`${installCommand} -D ${lang}\`.`, ) From 47e2cf69e1c711ec520d85176a9ed11e518a8283 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 19 Jan 2024 01:26:55 +0900 Subject: [PATCH 13/16] chore: add comment about require --- packages/vite/src/node/plugins/css.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 1be0e0b4b5a4f9..6ea9fbb7e3159f 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -2085,7 +2085,7 @@ const makeScssWorker = ( // additionalData can a function that is not cloneable but it won't be used options: SassStylePreprocessorOptions & { additionalData: undefined }, ) => { - // eslint-disable-next-line no-restricted-globals + // eslint-disable-next-line no-restricted-globals -- this function runs inside a cjs worker const sass: typeof Sass = require(sassPath) // eslint-disable-next-line no-restricted-globals const path = require('node:path') @@ -2310,7 +2310,7 @@ const makeLessWorker = ( const worker = new WorkerWithFallback( () => { - // eslint-disable-next-line no-restricted-globals + // eslint-disable-next-line no-restricted-globals -- this function runs inside a cjs worker const fsp = require('node:fs/promises') // eslint-disable-next-line no-restricted-globals const path = require('node:path') @@ -2367,7 +2367,7 @@ const makeLessWorker = ( // additionalData can a function that is not cloneable but it won't be used options: StylePreprocessorOptions & { additionalData: undefined }, ) => { - // eslint-disable-next-line no-restricted-globals + // eslint-disable-next-line no-restricted-globals -- this function runs inside a cjs worker const nodeLess: typeof Less = require(lessPath) const viteResolverPlugin = createViteLessPlugin( nodeLess, @@ -2477,7 +2477,7 @@ const makeStylWorker = (maxWorkers: number | undefined) => { // additionalData can a function that is not cloneable but it won't be used options: StylusStylePreprocessorOptions & { additionalData: undefined }, ) => { - // eslint-disable-next-line no-restricted-globals + // eslint-disable-next-line no-restricted-globals -- this function runs inside a cjs worker const nodeStylus: typeof Stylus = require(stylusPath) const ref = nodeStylus(content, options) From f548aad3f3a6708c56c882eec597ad6c7826a7ff Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 19 Jan 2024 01:32:27 +0900 Subject: [PATCH 14/16] chore: comment about shouldUseFake --- packages/vite/src/node/plugins/css.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 6ea9fbb7e3159f..929f745babbc92 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -2139,6 +2139,8 @@ const makeScssWorker = ( { parentFunctions: { internalImporter }, shouldUseFake(_sassPath, _data, options) { + // functions and importer is a function and is not serializable + // in that case, fallback to running in main thread return !!( (options.functions && Object.keys(options.functions).length > 0) || (options.importer && @@ -2391,6 +2393,8 @@ const makeLessWorker = ( { parentFunctions: { viteLessResolve }, shouldUseFake(_lessPath, _content, options) { + // plugins are a function and is not serializable + // in that case, fallback to running in main thread return options.plugins?.length > 0 }, max: maxWorkers, @@ -2504,6 +2508,8 @@ const makeStylWorker = (maxWorkers: number | undefined) => { }, { shouldUseFake(_stylusPath, _content, _root, options) { + // define can include functions and those are not serializable + // in that case, fallback to running in main thread return !!( options.define && Object.values(options.define).some((d) => typeof d === 'function') From f837bdf3eec35d7fe477976a80e54460804facc5 Mon Sep 17 00:00:00 2001 From: sapphi-red <49056869+sapphi-red@users.noreply.github.com> Date: Fri, 19 Jan 2024 02:07:04 +0900 Subject: [PATCH 15/16] feat: change `preprocessCSS` behavior --- packages/vite/src/node/index.ts | 6 +- packages/vite/src/node/plugins/css.ts | 89 ++++++++++++--------------- 2 files changed, 40 insertions(+), 55 deletions(-) diff --git a/packages/vite/src/node/index.ts b/packages/vite/src/node/index.ts index 01085fca7fe3ed..0c94465e1690be 100644 --- a/packages/vite/src/node/index.ts +++ b/packages/vite/src/node/index.ts @@ -12,11 +12,7 @@ export { createServer } from './server' export { preview } from './preview' export { build } from './build' export { optimizeDeps } from './optimizer' -export { - formatPostcssSourceMap, - createCSSPreprocessor, - preprocessCSS, -} from './plugins/css' +export { formatPostcssSourceMap, preprocessCSS } from './plugins/css' export { transformWithEsbuild } from './plugins/esbuild' export { buildErrorMessage } from './server/middlewares/error' export * from './publicUtils' diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 929f745babbc92..a65be349b2d74d 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -277,7 +277,13 @@ export function cssPlugin(config: ResolvedConfig): Plugin { removedPureCssFilesCache.set(config, new Map()) - preprocessorWorkerController = createPreprocessorWorkerController() + preprocessorWorkerController = createPreprocessorWorkerController( + normalizeMaxWorkers(config.css.preprocessorMaxWorkers), + ) + preprocessorWorkerControllerCache.set( + config, + preprocessorWorkerController, + ) }, buildEnd() { @@ -1125,7 +1131,6 @@ async function compileCSSPreprocessors( config.root, opts, atImportResolvers, - normalizeMaxWorkers(config.css.preprocessorMaxWorkers), ) if (preprocessResult.error) { throw preprocessResult.error @@ -1445,6 +1450,14 @@ const importPostcssImport = createCachedImport(() => import('postcss-import')) const importPostcssModules = createCachedImport(() => import('postcss-modules')) const importPostcss = createCachedImport(() => import('postcss')) +const preprocessorWorkerControllerCache = new WeakMap< + ResolvedConfig, + PreprocessorWorkerController +>() +let alwaysFakeWorkerWorkerControllerCache: + | PreprocessorWorkerController + | undefined + export interface PreprocessCSSResult { code: string map?: SourceMapInput @@ -1455,45 +1468,22 @@ export interface PreprocessCSSResult { /** * @experimental */ -export function createCSSPreprocessor(): { - process( - code: string, - filename: string, - config: ResolvedConfig, - ): Promise - close: () => void -} { - const preprocessorWorkerController = createPreprocessorWorkerController() - - return { - async process(code, filename, config) { - return await compileCSS( - filename, - code, - config, - preprocessorWorkerController, - ) - }, - close() { - preprocessorWorkerController.close() - }, - } -} - -/** - * @deprecated use createCSSPreprocessor instead - */ export async function preprocessCSS( code: string, filename: string, config: ResolvedConfig, ): Promise { - const p = createCSSPreprocessor() - try { - return await p.process(code, filename, config) - } finally { - p.close() + let workerController = preprocessorWorkerControllerCache.get(config) + + if (!workerController) { + // if workerController doesn't exist, create a workerController that always uses fake workers + // because fake workers doesn't require calling `.close` unlike real workers + alwaysFakeWorkerWorkerControllerCache ||= + createPreprocessorWorkerController(0) + workerController = alwaysFakeWorkerWorkerControllerCache } + + return await compileCSS(filename, code, config, workerController) } export async function formatPostcssSourceMap( @@ -1933,7 +1923,6 @@ type StylePreprocessor = { root: string, options: StylePreprocessorOptions, resolvers: CSSAtImportResolvers, - maxWorkers: number | undefined, ) => StylePreprocessorResults | Promise close: () => void } @@ -1944,7 +1933,6 @@ type SassStylePreprocessor = { root: string, options: SassStylePreprocessorOptions, resolvers: CSSAtImportResolvers, - maxWorkers: number | undefined, ) => StylePreprocessorResults | Promise close: () => void } @@ -1955,7 +1943,6 @@ type StylusStylePreprocessor = { root: string, options: StylusStylePreprocessorOptions, resolvers: CSSAtImportResolvers, - maxWorkers: number | undefined, ) => StylePreprocessorResults | Promise close: () => void } @@ -2153,7 +2140,9 @@ const makeScssWorker = ( return worker } -const scssProcessor = (): SassStylePreprocessor => { +const scssProcessor = ( + maxWorkers: number | undefined, +): SassStylePreprocessor => { const workerMap = new Map>() return { @@ -2162,7 +2151,7 @@ const scssProcessor = (): SassStylePreprocessor => { worker.stop() } }, - async process(source, root, options, resolvers, maxWorkers) { + async process(source, root, options, resolvers) { const sassPath = loadPreprocessorPath(PreprocessLang.sass, root) if (!workerMap.has(options.alias)) { @@ -2403,7 +2392,7 @@ const makeLessWorker = ( return worker } -const lessProcessor = (): StylePreprocessor => { +const lessProcessor = (maxWorkers: number | undefined): StylePreprocessor => { const workerMap = new Map>() return { @@ -2412,7 +2401,7 @@ const lessProcessor = (): StylePreprocessor => { worker.stop() } }, - async process(source, root, options, resolvers, maxWorkers) { + async process(source, root, options, resolvers) { const lessPath = loadPreprocessorPath(PreprocessLang.less, root) if (!workerMap.has(options.alias)) { @@ -2521,7 +2510,9 @@ const makeStylWorker = (maxWorkers: number | undefined) => { return worker } -const stylProcessor = (): StylusStylePreprocessor => { +const stylProcessor = ( + maxWorkers: number | undefined, +): StylusStylePreprocessor => { const workerMap = new Map>() return { @@ -2530,7 +2521,7 @@ const stylProcessor = (): StylusStylePreprocessor => { worker.stop() } }, - async process(source, root, options, resolvers, maxWorkers) { + async process(source, root, options, resolvers) { const stylusPath = loadPreprocessorPath(PreprocessLang.stylus, root) if (!workerMap.has(options.alias)) { @@ -2632,24 +2623,22 @@ async function getSource( } } -const createPreprocessorWorkerController = () => { - const scss = scssProcessor() - const less = lessProcessor() - const styl = stylProcessor() +const createPreprocessorWorkerController = (maxWorkers: number | undefined) => { + const scss = scssProcessor(maxWorkers) + const less = lessProcessor(maxWorkers) + const styl = stylProcessor(maxWorkers) const sassProcess: StylePreprocessor['process'] = ( source, root, options, resolvers, - maxWorkers, ) => { return scss.process( source, root, { ...options, indentedSyntax: true }, resolvers, - maxWorkers, ) } From 446eb4f2d5e9a45f16dc18616e3279f40d839d38 Mon Sep 17 00:00:00 2001 From: patak Date: Mon, 22 Jan 2024 11:21:42 +0100 Subject: [PATCH 16/16] chore: wording --- docs/config/shared-options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config/shared-options.md b/docs/config/shared-options.md index 4bdebf84b0eee8..d2eae4ec2d9722 100644 --- a/docs/config/shared-options.md +++ b/docs/config/shared-options.md @@ -267,7 +267,7 @@ export default defineConfig({ - **Type:** `number | true` - **Default:** `0` (does not create any workers and run in the main thread) -If this option is set, preprocessors will run in workers when possible. `true` means the number of CPUs minus 1. +If this option is set, CSS preprocessors will run in workers when possible. `true` means the number of CPUs minus 1. ## css.devSourcemap