diff --git a/.eslintrc.json b/.eslintrc.json index 16517610ab..29a86ef1a4 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -558,10 +558,12 @@ { "files": [ "ext/js/core/api-map.js", + "ext/js/core/event-listener-collection.js", "ext/js/core/extension-error.js", "ext/js/core/json.js", "ext/js/data/anki-note-data-creator.js", "ext/js/dictionary/dictionary-data-util.js", + "ext/js/display/display-content-manager.js", "ext/js/display/pronunciation-generator.js", "ext/js/display/structured-content-generator.js", "ext/js/dom/css-style-applier.js", @@ -582,6 +584,7 @@ }, { "files": [ + "ext/js/core/api-map.js", "ext/js/core/event-dispatcher.js", "ext/js/core/extension-error.js", "ext/js/core/json.js", diff --git a/benches/jsconfig.json b/benches/jsconfig.json index e982a05525..9cf87cdd1a 100644 --- a/benches/jsconfig.json +++ b/benches/jsconfig.json @@ -23,11 +23,18 @@ }, "types": [ "chrome", + "dom-webcodecs", "firefox-webext-browser", "handlebars", "jszip", "parse5", "wanakana" + ], + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable", + "WebWorker" ] }, "include": [ diff --git a/benches/translator.bench.js b/benches/translator.bench.js index 1b621af65f..aac1b62cfb 100644 --- a/benches/translator.bench.js +++ b/benches/translator.bench.js @@ -21,8 +21,11 @@ import path from 'path'; import {bench, describe} from 'vitest'; import {parseJson} from '../dev/json.js'; import {createTranslatorContext} from '../test/fixtures/translator-test.js'; +import {setupStubs} from '../test/utilities/database.js'; import {createFindKanjiOptions, createFindTermsOptions} from '../test/utilities/translator.js'; +setupStubs(); + const dirname = path.dirname(fileURLToPath(import.meta.url)); const dictionaryName = 'Test Dictionary 2'; const {translator} = await createTranslatorContext(path.join(dirname, '..', 'test', 'data/dictionaries/valid-dictionary1'), dictionaryName); diff --git a/dev/build-libs.js b/dev/build-libs.js index 0d42f3404c..6bafc5e356 100644 --- a/dev/build-libs.js +++ b/dev/build-libs.js @@ -20,13 +20,27 @@ import Ajv from 'ajv'; import standaloneCode from 'ajv/dist/standalone/index.js'; import esbuild from 'esbuild'; import fs from 'fs'; +import {createRequire} from 'module'; import path from 'path'; import {fileURLToPath} from 'url'; import {parseJson} from './json.js'; +const require = createRequire(import.meta.url); + const dirname = path.dirname(fileURLToPath(import.meta.url)); const extDir = path.join(dirname, '..', 'ext'); +/** + * @param {string} out + */ +async function copyWasm(out) { + // copy from node modules '@resvg/resvg-wasm/index_bg.wasm' to out + const resvgWasmPath = path.dirname(require.resolve('@resvg/resvg-wasm')); + const wasmPath = path.join(resvgWasmPath, 'index_bg.wasm'); + fs.copyFileSync(wasmPath, path.join(out, 'resvg.wasm')); +} + + /** * @param {string} scriptPath */ @@ -79,4 +93,6 @@ export async function buildLibs() { const patchedModuleCode = "// @ts-nocheck\nimport {ucs2length} from './ucs2length.js';" + moduleCode.replaceAll('require("ajv/dist/runtime/ucs2length").default', 'ucs2length'); fs.writeFileSync(path.join(extDir, 'lib/validate-schemas.js'), patchedModuleCode); + + await copyWasm(path.join(extDir, 'lib')); } diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json index a1b0f8e92b..0dd2a22f6b 100644 --- a/dev/data/manifest-variants.json +++ b/dev/data/manifest-variants.json @@ -102,7 +102,8 @@ "resources": [ "popup.html", "template-renderer.html", - "js/*" + "js/*", + "lib/resvg.wasm" ], "matches": [ "" @@ -110,7 +111,7 @@ } ], "content_security_policy": { - "extension_pages": "default-src 'self'; img-src blob: 'self'; style-src 'self' 'unsafe-inline'; media-src *; connect-src *", + "extension_pages": "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; img-src blob: 'self'; style-src 'self' 'unsafe-inline'; media-src *; connect-src *", "sandbox": "sandbox allow-scripts; default-src 'self'; script-src 'self' 'unsafe-eval'; style-src 'unsafe-inline'" } }, @@ -238,7 +239,7 @@ "content_security_policy", "extension_pages" ], - "value": "default-src 'self'; script-src 'self'; img-src blob: 'self'; style-src 'self' 'unsafe-inline'; media-src *; connect-src *" + "value": "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; img-src blob: 'self'; style-src 'self' 'unsafe-inline'; media-src *; connect-src *" }, { "action": "set", diff --git a/dev/data/structured-content-overrides.css b/dev/data/structured-content-overrides.css index c792cbb684..0878eede20 100644 --- a/dev/data/structured-content-overrides.css +++ b/dev/data/structured-content-overrides.css @@ -28,23 +28,10 @@ .gloss-image-link:hover { /* remove-rule */ } -.gloss-image-container-overlay { - font-size: initial; - line-height: initial; - color: initial; -} -.gloss-image-background { - background-color: currentColor; -} :root[data-browser=firefox] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image, -:root[data-browser=firefox] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image-background, -:root[data-browser=firefox-mobile] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image, -:root[data-browser=firefox-mobile] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image-background { +:root[data-browser=firefox-mobile] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image { /* remove-rule */ } -.gloss-image-link-text { - line-height: initial; -} .gloss-sc-thead, .gloss-sc-tfoot, .gloss-sc-th { diff --git a/dev/jsconfig.json b/dev/jsconfig.json index 253424f8b0..211201062f 100644 --- a/dev/jsconfig.json +++ b/dev/jsconfig.json @@ -53,7 +53,14 @@ "assert", "css", "chrome", - "ajv" + "ajv", + "dom-webcodecs" + ], + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable", + "WebWorker" ] }, "include": [ diff --git a/dev/lib/resvg-wasm.js b/dev/lib/resvg-wasm.js new file mode 100644 index 0000000000..6c969bafd6 --- /dev/null +++ b/dev/lib/resvg-wasm.js @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023-2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +export * from '@resvg/resvg-wasm'; diff --git a/ext/css/display.css b/ext/css/display.css index 950007c911..c44c3f26be 100644 --- a/ext/css/display.css +++ b/ext/css/display.css @@ -832,6 +832,8 @@ button.action-button:active { .entry { padding: var(--entry-vertical-padding) var(--entry-horizontal-padding); position: relative; + content-visibility: auto; + contain-intrinsic-height: auto 500px; } .entry+.entry { border-top: var(--thin-border-size) solid var(--light-border-color); diff --git a/ext/css/structured-content.css b/ext/css/structured-content.css index 05bce018f7..a79d9ddc4e 100644 --- a/ext/css/structured-content.css +++ b/ext/css/structured-content.css @@ -19,14 +19,6 @@ /* Glossary images */ .gloss-image-container { display: inline-block; - white-space: nowrap; - max-width: 100%; - max-height: 100vh; - position: relative; - vertical-align: top; - line-height: 0; - font-size: calc(1em / var(--font-size-no-units)); - overflow: hidden; } .gloss-image-link[data-background=true]>.gloss-image-container { background-color: var(--gloss-image-background-color); @@ -45,112 +37,34 @@ .gloss-image-link[href]:hover { cursor: pointer; } -.gloss-image-container-overlay { - position: absolute; - left: 0; - top: 0; - width: 100%; - height: 100%; - font-size: calc(1em * var(--font-size-no-units)); - line-height: var(--line-height); - display: table; - table-layout: fixed; - white-space: normal; - color: var(--text-color-light3); -} -.gloss-image-link[data-has-image=true][data-image-load-state=load-error] .gloss-image-container-overlay::after { - content: 'Image failed to load'; - display: table-cell; - width: 100%; - height: 100%; - vertical-align: middle; - text-align: center; - padding: 0.25em; -} -.gloss-image-background { - --image: none; - - position: absolute; - left: 0; - top: 0; - width: 100%; - height: 100%; - background-color: var(--text-color); - -webkit-mask-repeat: no-repeat; - -webkit-mask-position: center center; - -webkit-mask-mode: alpha; - -webkit-mask-size: contain; - -webkit-mask-image: var(--image); - mask-repeat: no-repeat; - mask-position: center center; - mask-mode: alpha; - mask-size: contain; - mask-image: var(--image); -} .gloss-image { display: inline-block; vertical-align: top; object-fit: contain; border: none; outline: none; -} -.gloss-image-link[data-has-aspect-ratio=true] .gloss-image { - position: absolute; - left: 0; - top: 0; width: 100%; - height: 100%; -} -.gloss-image:not([src]) { - display: none; } -.gloss-image-link[data-image-rendering=pixelated] .gloss-image, -.gloss-image-link[data-image-rendering=pixelated] .gloss-image-background { +.gloss-image-link[data-image-rendering=pixelated] .gloss-image { image-rendering: auto; image-rendering: -moz-crisp-edges; image-rendering: -webkit-optimize-contrast; image-rendering: pixelated; image-rendering: crisp-edges; } -.gloss-image-link[data-image-rendering=crisp-edges] .gloss-image, -.gloss-image-link[data-image-rendering=crisp-edges] .gloss-image-background { +.gloss-image-link[data-image-rendering=crisp-edges] .gloss-image { image-rendering: auto; image-rendering: -moz-crisp-edges; image-rendering: -webkit-optimize-contrast; image-rendering: crisp-edges; } :root[data-browser=firefox] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image, -:root[data-browser=firefox] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image-background, -:root[data-browser=firefox-mobile] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image, -:root[data-browser=firefox-mobile] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image-background { +:root[data-browser=firefox-mobile] .gloss-image-link[data-image-rendering=crisp-edges] .gloss-image { image-rendering: auto; } -.gloss-image-link[data-has-aspect-ratio=true] .gloss-image-sizer { - display: inline-block; - width: 0; - vertical-align: top; - font-size: 0; -} -.gloss-image-link-text { - display: none; - line-height: var(--line-height); -} -.gloss-image-link-text::before { - content: '['; -} -.gloss-image-link-text::after { - content: ']'; -} -.gloss-image-description { - display: block; - white-space: pre-line; -} .gloss-image-link[data-appearance=monochrome] .gloss-image { - opacity: 0; -} -.gloss-image-link:not([data-appearance=monochrome]) .gloss-image-background { - display: none; + filter: grayscale(1); } .gloss-image-link[data-size-units=em] .gloss-image-container { @@ -189,14 +103,6 @@ :root[data-glossary-layout-mode=compact] .gloss-image-link[data-collapsible=true]:focus .gloss-image-container { display: block; } -.gloss-image-link[data-collapsed=true] .gloss-image-link-text, -:root[data-glossary-layout-mode=compact] .gloss-image-link[data-collapsible=true] .gloss-image-link-text { - display: inline; -} -.gloss-image-link[data-collapsed=true]~.gloss-image-description, -:root[data-glossary-layout-mode=compact] .gloss-image-description { - display: inline; -} /* Links */ diff --git a/ext/data/structured-content-style.json b/ext/data/structured-content-style.json index a1110dce18..47b1e99513 100644 --- a/ext/data/structured-content-style.json +++ b/ext/data/structured-content-style.json @@ -3,13 +3,6 @@ "selectors": [".gloss-image-container"], "styles": [ ["display", "inline-block"], - ["white-space", "nowrap"], - ["max-width", "100%"], - ["max-height", "100vh"], - ["position", "relative"], - ["vertical-align", "top"], - ["line-height", "0"], - ["overflow", "hidden"], ["font-size", "1px"] ] }, @@ -30,56 +23,6 @@ ["cursor", "pointer"] ] }, - { - "selectors": [".gloss-image-container-overlay"], - "styles": [ - ["position", "absolute"], - ["left", "0"], - ["top", "0"], - ["width", "100%"], - ["height", "100%"], - ["display", "table"], - ["table-layout", "fixed"], - ["white-space", "normal"], - ["font-size", "initial"], - ["line-height", "initial"], - ["color", "initial"] - ] - }, - { - "selectors": [".gloss-image-link[data-has-image=true][data-image-load-state=load-error] .gloss-image-container-overlay::after"], - "styles": [ - ["content", "'Image failed to load'"], - ["display", "table-cell"], - ["width", "100%"], - ["height", "100%"], - ["vertical-align", "middle"], - ["text-align", "center"], - ["padding", "0.25em"] - ] - }, - { - "selectors": [".gloss-image-background"], - "styles": [ - ["--image", "none"], - ["position", "absolute"], - ["left", "0"], - ["top", "0"], - ["width", "100%"], - ["height", "100%"], - ["-webkit-mask-repeat", "no-repeat"], - ["-webkit-mask-position", "center center"], - ["-webkit-mask-mode", "alpha"], - ["-webkit-mask-size", "contain"], - ["-webkit-mask-image", "var(--image)"], - ["mask-repeat", "no-repeat"], - ["mask-position", "center center"], - ["mask-mode", "alpha"], - ["mask-size", "contain"], - ["mask-image", "var(--image)"], - ["background-color", "currentColor"] - ] - }, { "selectors": [".gloss-image"], "styles": [ @@ -87,30 +30,12 @@ ["vertical-align", "top"], ["object-fit", "contain"], ["border", "none"], - ["outline", "none"] + ["outline", "none"], + ["width", "100%"] ] }, { - "selectors": [".gloss-image-link[data-has-aspect-ratio=true] .gloss-image"], - "styles": [ - ["position", "absolute"], - ["left", "0"], - ["top", "0"], - ["width", "100%"], - ["height", "100%"] - ] - }, - { - "selectors": [".gloss-image:not([src])"], - "styles": [ - ["display", "none"] - ] - }, - { - "selectors": [ - ".gloss-image-link[data-image-rendering=pixelated] .gloss-image", - ".gloss-image-link[data-image-rendering=pixelated] .gloss-image-background" - ], + "selectors": [".gloss-image-link[data-image-rendering=pixelated] .gloss-image"], "styles": [ ["image-rendering", "auto"], ["image-rendering", "-moz-crisp-edges"], @@ -120,10 +45,7 @@ ] }, { - "selectors": [ - ".gloss-image-link[data-image-rendering=crisp-edges] .gloss-image", - ".gloss-image-link[data-image-rendering=crisp-edges] .gloss-image-background" - ], + "selectors": [".gloss-image-link[data-image-rendering=crisp-edges] .gloss-image"], "styles": [ ["image-rendering", "auto"], ["image-rendering", "-moz-crisp-edges"], @@ -131,51 +53,10 @@ ["image-rendering", "crisp-edges"] ] }, - { - "selectors": [".gloss-image-link[data-has-aspect-ratio=true] .gloss-image-sizer"], - "styles": [ - ["display", "inline-block"], - ["width", "0"], - ["vertical-align", "top"], - ["font-size", "0"] - ] - }, - { - "selectors": [".gloss-image-link-text"], - "styles": [ - ["display", "none"], - ["line-height", "initial"] - ] - }, - { - "selectors": [".gloss-image-link-text::before"], - "styles": [ - ["content", "'['"] - ] - }, - { - "selectors": [".gloss-image-link-text::after"], - "styles": [ - ["content", "']'"] - ] - }, - { - "selectors": [".gloss-image-description"], - "styles": [ - ["display", "block"], - ["white-space", "pre-line"] - ] - }, { "selectors": [".gloss-image-link[data-appearance=monochrome] .gloss-image"], "styles": [ - ["opacity", "0"] - ] - }, - { - "selectors": [".gloss-image-link:not([data-appearance=monochrome]) .gloss-image-background"], - "styles": [ - ["display", "none"] + ["filter", "grayscale(1)"] ] }, { @@ -275,24 +156,6 @@ ["display", "block"] ] }, - { - "selectors": [ - ".gloss-image-link[data-collapsed=true] .gloss-image-link-text", - ":root[data-glossary-layout-mode=compact] .gloss-image-link[data-collapsible=true] .gloss-image-link-text" - ], - "styles": [ - ["display", "inline"] - ] - }, - { - "selectors": [ - ".gloss-image-link[data-collapsed=true]~.gloss-image-description", - ":root[data-glossary-layout-mode=compact] .gloss-image-description" - ], - "styles": [ - ["display", "inline"] - ] - }, { "selectors": [".gloss-link-external-icon"], "styles": [ diff --git a/ext/fonts/NotoSansJP-Regular.ttf b/ext/fonts/NotoSansJP-Regular.ttf new file mode 100644 index 0000000000..b2dad730d7 Binary files /dev/null and b/ext/fonts/NotoSansJP-Regular.ttf differ diff --git a/ext/js/application.js b/ext/js/application.js index fc532a6d8f..b60567488a 100644 --- a/ext/js/application.js +++ b/ext/js/application.js @@ -190,10 +190,35 @@ export class Application extends EventDispatcher { * @param {(application: Application) => (Promise)} mainFunction */ static async main(waitForDom, mainFunction) { + /** @type {MessagePort | null} */ + // If this is Firefox, we don't have a service worker and can't postMessage, + // so we temporarily create a SharedWorker in order to establish a MessageChannel + // which we can use to postMessage with the backend. + // This can only be done in the extension context (aka iframe within popup), + // not in the content script context. + const backendPort = (!('serviceWorker' in navigator)) && window.location.protocol === new URL(import.meta.url).protocol ? + (() => { + const sharedWorkerBridge = new SharedWorker(new URL('comm/shared-worker-bridge.js', import.meta.url), {type: 'module'}); + const backendChannel = new MessageChannel(); + sharedWorkerBridge.port.postMessage({action: 'connectToBackend1'}, [backendChannel.port1]); + sharedWorkerBridge.port.close(); + return backendChannel.port2; + })() : + null; + const webExtension = new WebExtension(); log.configure(webExtension.extensionName); - const api = new API(webExtension); + + const mediaDrawingWorkerToBackendChannel = new MessageChannel(); + const mediaDrawingWorker = window.location.protocol === new URL(import.meta.url).protocol ? new Worker(new URL('display/media-drawing-worker.js', import.meta.url), {type: 'module'}) : null; + mediaDrawingWorker?.postMessage({action: 'connectToDatabaseWorker'}, [mediaDrawingWorkerToBackendChannel.port2]); + + const api = new API(webExtension, mediaDrawingWorker, backendPort); await waitForBackendReady(webExtension); + if (mediaDrawingWorker !== null) { + api.connectToDatabaseWorker(mediaDrawingWorkerToBackendChannel.port1); + } + const {tabId, frameId} = await api.frameInformationGet(); const crossFrameApi = new CrossFrameAPI(api, tabId, frameId); crossFrameApi.prepare(); diff --git a/ext/js/background/backend.js b/ext/js/background/backend.js index efb40ed470..647b22e99f 100644 --- a/ext/js/background/backend.js +++ b/ext/js/background/backend.js @@ -186,6 +186,12 @@ export class Backend { ['openCrossFramePort', this._onApiOpenCrossFramePort.bind(this)], ['getLanguageSummaries', this._onApiGetLanguageSummaries.bind(this)], ]); + + /** @type {import('api').PmApiMap} */ + this._pmApiMap = createApiMap([ + ['connectToDatabaseWorker', this._onPmConnectToDatabaseWorker.bind(this)], + ['registerOffscreenPort', this._onPmApiRegisterOffscreenPort.bind(this)], + ]); /* eslint-enable @stylistic/no-multi-spaces */ /** @type {Map void>} */ @@ -240,6 +246,9 @@ export class Backend { const onMessage = this._onMessageWrapper.bind(this); chrome.runtime.onMessage.addListener(onMessage); + // On Chrome, this is for receiving messages sent with navigator.serviceWorker, which has the benefit of being able to transfer objects, but doesn't accept callbacks + (/** @type {ServiceWorkerGlobalScope & typeof globalThis} */ (globalThis)).addEventListener('message', this._onPmMessage.bind(this)); + if (this._canObservePermissionsChanges()) { const onPermissionsChanged = this._onWebExtensionEventWrapper(this._onPermissionsChanged.bind(this)); chrome.permissions.onAdded.addListener(onPermissionsChanged); @@ -249,6 +258,20 @@ export class Backend { chrome.runtime.onInstalled.addListener(this._onInstalled.bind(this)); } + /** @type {import('api').PmApiHandler<'connectToDatabaseWorker'>} */ + async _onPmConnectToDatabaseWorker(_params, ports) { + if (ports !== null && ports.length > 0) { + await this._dictionaryDatabase.connectToDatabaseWorker(ports[0]); + } + } + + /** @type {import('api').PmApiHandler<'registerOffscreenPort'>} */ + async _onPmApiRegisterOffscreenPort(_params, ports) { + if (ports !== null && ports.length > 0) { + await this._offscreen?.registerOffscreenPort(ports[0]); + } + } + /** * @returns {Promise} */ @@ -273,6 +296,16 @@ export class Backend { } this._clipboardReader.browser = this._environment.getInfo().browser; + // if this is Firefox and therefore not running in Service Worker, we need to use a SharedWorker to setup a MessageChannel to postMessage with the popup + if (self.constructor.name === 'Window') { + const sharedWorkerBridge = new SharedWorker(new URL('../comm/shared-worker-bridge.js', import.meta.url), {type: 'module'}); + sharedWorkerBridge.port.postMessage({action: 'registerBackendPort'}); + sharedWorkerBridge.port.addEventListener('message', (/** @type {MessageEvent} */ e) => { + // connectToBackend2 + e.ports[0].onmessage = this._onPmMessage.bind(this); + }); + sharedWorkerBridge.port.start(); + } try { await this._dictionaryDatabase.prepare(); } catch (e) { @@ -408,6 +441,15 @@ export class Backend { return invokeApiMapHandler(this._apiMap, action, params, [sender], callback); } + /** + * @param {MessageEvent} event + * @returns {boolean} + */ + _onPmMessage(event) { + const {action, params} = event.data; + return invokeApiMapHandler(this._pmApiMap, action, params, [event.ports], () => {}); + } + /** * @param {chrome.tabs.ZoomChangeInfo} event */ diff --git a/ext/js/background/offscreen-proxy.js b/ext/js/background/offscreen-proxy.js index 7150cc2922..d94874e498 100644 --- a/ext/js/background/offscreen-proxy.js +++ b/ext/js/background/offscreen-proxy.js @@ -22,12 +22,16 @@ import {base64ToArrayBuffer} from '../data/array-buffer-util.js'; /** * This class is responsible for creating and communicating with an offscreen document. - * This offscreen document is used to solve two issues: + * This offscreen document is used to solve three issues: * * - Provide clipboard access for the `ClipboardReader` class in the context of a MV3 extension. * The background service workers doesn't have access a webpage to read the clipboard from, * so it must be done in the offscreen page. * + * - Create a worker for image rendering, which both selects the images from the database, + * decodes/rasterizes them, and then sends (= postMessage transfers) them back to a worker + * in the popup to be rendered onto OffscreenCanvas. + * * - Provide a longer lifetime for the dictionary database. The background service worker can be * terminated by the web browser, which means that when it restarts, it has to go through its * initialization process again. This initialization process can take a non-trivial amount of @@ -55,6 +59,9 @@ export class OffscreenProxy { this._webExtension = webExtension; /** @type {?Promise} */ this._creatingOffscreen = null; + + /** @type {?MessagePort} */ + this._currentOffscreenPort = null; } /** @@ -62,6 +69,7 @@ export class OffscreenProxy { */ async prepare() { if (await this._hasOffscreenDocument()) { + void this.sendMessagePromise({action: 'createAndRegisterPortOffscreen'}); return; } if (this._creatingOffscreen) { @@ -133,6 +141,30 @@ export class OffscreenProxy { } return response.result; } + + /** + * @param {MessagePort} port + */ + async registerOffscreenPort(port) { + if (this._currentOffscreenPort) { + this._currentOffscreenPort.close(); + } + this._currentOffscreenPort = port; + } + + /** + * When you need to transfer Transferable objects, you can use this method which uses postMessage over the MessageChannel port established with the offscreen document. + * @template {import('offscreen').McApiNames} TMessageType + * @param {import('offscreen').McApiMessage} message + * @param {Transferable[]} transfers + */ + sendMessageViaPort(message, transfers) { + if (this._currentOffscreenPort !== null) { + this._currentOffscreenPort.postMessage(message, transfers); + } else { + void this.sendMessagePromise({action: 'createAndRegisterPortOffscreen'}); + } + } } export class DictionaryDatabaseProxy { @@ -173,6 +205,14 @@ export class DictionaryDatabaseProxy { const serializedMedia = /** @type {import('dictionary-database').Media[]} */ (await this._offscreen.sendMessagePromise({action: 'databaseGetMediaOffscreen', params: {targets}})); return serializedMedia.map((m) => ({...m, content: base64ToArrayBuffer(m.content)})); } + + /** + * @param {MessagePort} port + * @returns {Promise} + */ + async connectToDatabaseWorker(port) { + this._offscreen.sendMessageViaPort({action: 'connectToDatabaseWorker'}, [port]); + } } export class TranslatorProxy { diff --git a/ext/js/background/offscreen.js b/ext/js/background/offscreen.js index 5e901005ef..cbf943f49c 100644 --- a/ext/js/background/offscreen.js +++ b/ext/js/background/offscreen.js @@ -16,10 +16,12 @@ * along with this program. If not, see . */ +import {API} from '../comm/api.js'; import {ClipboardReader} from '../comm/clipboard-reader.js'; import {createApiMap, invokeApiMapHandler} from '../core/api-map.js'; import {arrayBufferToBase64} from '../data/array-buffer-util.js'; import {DictionaryDatabase} from '../dictionary/dictionary-database.js'; +import {WebExtension} from '../extension/web-extension.js'; import {Translator} from '../language/translator.js'; /** @@ -42,32 +44,44 @@ export class Offscreen { '#clipboard-rich-content-paste-target', ); - /* eslint-disable @stylistic/no-multi-spaces */ /** @type {import('offscreen').ApiMap} */ this._apiMap = createApiMap([ - ['clipboardGetTextOffscreen', this._getTextHandler.bind(this)], - ['clipboardGetImageOffscreen', this._getImageHandler.bind(this)], - ['clipboardSetBrowserOffscreen', this._setClipboardBrowser.bind(this)], - ['databasePrepareOffscreen', this._prepareDatabaseHandler.bind(this)], - ['getDictionaryInfoOffscreen', this._getDictionaryInfoHandler.bind(this)], - ['databasePurgeOffscreen', this._purgeDatabaseHandler.bind(this)], - ['databaseGetMediaOffscreen', this._getMediaHandler.bind(this)], - ['translatorPrepareOffscreen', this._prepareTranslatorHandler.bind(this)], - ['findKanjiOffscreen', this._findKanjiHandler.bind(this)], - ['findTermsOffscreen', this._findTermsHandler.bind(this)], - ['getTermFrequenciesOffscreen', this._getTermFrequenciesHandler.bind(this)], - ['clearDatabaseCachesOffscreen', this._clearDatabaseCachesHandler.bind(this)], + ['clipboardGetTextOffscreen', this._getTextHandler.bind(this)], + ['clipboardGetImageOffscreen', this._getImageHandler.bind(this)], + ['clipboardSetBrowserOffscreen', this._setClipboardBrowser.bind(this)], + ['databasePrepareOffscreen', this._prepareDatabaseHandler.bind(this)], + ['getDictionaryInfoOffscreen', this._getDictionaryInfoHandler.bind(this)], + ['databasePurgeOffscreen', this._purgeDatabaseHandler.bind(this)], + ['databaseGetMediaOffscreen', this._getMediaHandler.bind(this)], + ['translatorPrepareOffscreen', this._prepareTranslatorHandler.bind(this)], + ['findKanjiOffscreen', this._findKanjiHandler.bind(this)], + ['findTermsOffscreen', this._findTermsHandler.bind(this)], + ['getTermFrequenciesOffscreen', this._getTermFrequenciesHandler.bind(this)], + ['clearDatabaseCachesOffscreen', this._clearDatabaseCachesHandler.bind(this)], + ['createAndRegisterPortOffscreen', this._createAndRegisterPort.bind(this)], ]); /* eslint-enable @stylistic/no-multi-spaces */ + /** @type {import('offscreen').McApiMap} */ + this._mcApiMap = createApiMap([ + ['connectToDatabaseWorker', this._connectToDatabaseWorkerHandler.bind(this)], + ]); + /** @type {?Promise} */ this._prepareDatabasePromise = null; + + /** + * @type {API} + */ + this._api = new API(new WebExtension()); } /** */ prepare() { chrome.runtime.onMessage.addListener(this._onMessage.bind(this)); + navigator.serviceWorker.addEventListener('controllerchange', this._createAndRegisterPort.bind(this)); + this._createAndRegisterPort(); } /** @type {import('offscreen').ApiHandler<'clipboardGetTextOffscreen'>} */ @@ -130,8 +144,8 @@ export class Offscreen { const enabledDictionaryMap = new Map(options.enabledDictionaryMap); const excludeDictionaryDefinitions = ( options.excludeDictionaryDefinitions !== null ? - new Set(options.excludeDictionaryDefinitions) : - null + new Set(options.excludeDictionaryDefinitions) : + null ); const textReplacements = options.textReplacements.map((group) => { if (group === null) { return null; } @@ -166,4 +180,26 @@ export class Offscreen { _onMessage({action, params}, _sender, callback) { return invokeApiMapHandler(this._apiMap, action, params, [], callback); } + + /** + * + */ + _createAndRegisterPort() { + const mc = new MessageChannel(); + mc.port1.onmessage = this._onMcMessage.bind(this); + this._api.registerOffscreenPort([mc.port2]); + } + + /** @type {import('offscreen').McApiHandler<'connectToDatabaseWorker'>} */ + async _connectToDatabaseWorkerHandler(_params, ports) { + await this._dictionaryDatabase.connectToDatabaseWorker(ports[0]); + } + + /** + * @param {MessageEvent} event + */ + _onMcMessage(event) { + const {action, params} = event.data; + invokeApiMapHandler(this._mcApiMap, action, params, [event.ports], () => {}); + } } diff --git a/ext/js/comm/api.js b/ext/js/comm/api.js index 7668a0d175..0d95383d18 100644 --- a/ext/js/comm/api.js +++ b/ext/js/comm/api.js @@ -17,14 +17,23 @@ */ import {ExtensionError} from '../core/extension-error.js'; +import {log} from '../core/log.js'; export class API { /** * @param {import('../extension/web-extension.js').WebExtension} webExtension + * @param {Worker?} mediaDrawingWorker + * @param {MessagePort?} backendPort */ - constructor(webExtension) { + constructor(webExtension, mediaDrawingWorker = null, backendPort = null) { /** @type {import('../extension/web-extension.js').WebExtension} */ this._webExtension = webExtension; + + /** @type {Worker?} */ + this._mediaDrawingWorker = mediaDrawingWorker; + + /** @type {MessagePort?} */ + this._backendPort = backendPort; } /** @@ -254,6 +263,14 @@ export class API { return this._invoke('getMedia', {targets}); } + /** + * @param {import('api').PmApiParam<'drawMedia', 'requests'>} requests + * @param {Transferable[]} transferables + */ + drawMedia(requests, transferables) { + this._mediaDrawingWorker?.postMessage({action: 'drawMedia', params: {requests}}, transferables); + } + /** * @param {import('api').ApiParam<'logGenericErrorBackend', 'error'>} error * @param {import('api').ApiParam<'logGenericErrorBackend', 'level'>} level @@ -364,6 +381,20 @@ export class API { return this._invoke('openCrossFramePort', {targetTabId, targetFrameId}); } + /** + * @param {Transferable[]} transferables + */ + registerOffscreenPort(transferables) { + this._pmInvoke('registerOffscreenPort', void 0, transferables); + } + + /** + * @param {MessagePort} port + */ + connectToDatabaseWorker(port) { + this._pmInvoke('connectToDatabaseWorker', void 0, [port]); + } + /** * @returns {Promise>} */ @@ -405,4 +436,31 @@ export class API { } }); } + + /** + * @template {import('api').PmApiNames} TAction + * @template {import('api').PmApiParams} TParams + * @param {TAction} action + * @param {TParams} params + * @param {Transferable[]} transferables + */ + _pmInvoke(action, params, transferables) { + // on firefox, there is no service worker, so we instead use a MessageChannel which is established + // via a handshake via a SharedWorker + if (!('serviceWorker' in navigator)) { + if (this._backendPort === null) { + log.error('no backend port available'); + return; + } + this._backendPort.postMessage({action, params}, transferables); + } else { + void navigator.serviceWorker.ready.then((swr) => { + if (swr.active !== null) { + swr.active.postMessage({action, params}, transferables); + } else { + log.error(`[${self.constructor.name}] no active service worker`); + } + }); + } + } } diff --git a/ext/js/comm/shared-worker-bridge.js b/ext/js/comm/shared-worker-bridge.js new file mode 100644 index 0000000000..32b5f8a230 --- /dev/null +++ b/ext/js/comm/shared-worker-bridge.js @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {createApiMap, invokeApiMapHandler} from '../core/api-map.js'; +import {log} from '../core/log.js'; + +/** + * This serves as a bridge between the application and the backend. + * It is designed to have extremely short lifetime on the application side, + * as otherwise it will stay alive across extension updates (which only restart + * the backend) which can lead to extremely difficult to debug situations where + * the bridge is running an old version of the code. + * + * All it does is broker a handshake between the application and the backend, + * where they establish a connection between each other with a MessageChannel. + * + * # On backend startup + * backend + * ↓↓<"registerBackendPort" via SharedWorker.port.postMessage>↓↓ + * bridge: store the port in state + * + * # On application startup + * application: create a new MessageChannel, bind event listeners to one of the ports, and send the other port to the bridge + * ↓↓<"connectToBackend1" via SharedWorker.port.postMessage>↓↓ + * bridge + * ↓↓<"connectToBackend2" via MessageChannel.port.postMessage which is stored in state from backend startup phase>↓↓ + * backend: bind event listeners to the other port + */ +export class SharedWorkerBridge { + constructor() { + /** @type {MessagePort?} */ + this._backendPort = null; + + /** @type {import('shared-worker').ApiMap} */ + this._apiMap = createApiMap([ + ['registerBackendPort', this._onRegisterBackendPort.bind(this)], + ['connectToBackend1', this._onConnectToBackend1.bind(this)], + ]); + } + + /** + * + */ + prepare() { + addEventListener('connect', (connectEvent) => { + const interlocutorPort = (/** @type {MessageEvent} */ (connectEvent)).ports[0]; + interlocutorPort.addEventListener('message', (/** @type {MessageEvent} */ event) => { + const {action, params} = event.data; + return invokeApiMapHandler(this._apiMap, action, params, [interlocutorPort, event.ports], () => {}); + }); + interlocutorPort.start(); + }); + } + + /** @type {import('shared-worker').ApiHandler<'registerBackendPort'>} */ + _onRegisterBackendPort(_params, interlocutorPort, _ports) { + this._backendPort = interlocutorPort; + } + + /** @type {import('shared-worker').ApiHandler<'connectToBackend1'>} */ + _onConnectToBackend1(_params, _interlocutorPort, ports) { + if (this._backendPort !== null) { + this._backendPort.postMessage(void 0, [ports[0]]); // connectToBackend2 + } else { + log.error('SharedWorkerBridge: backend port is not registered'); + } + } +} + +const bridge = new SharedWorkerBridge(); +bridge.prepare(); diff --git a/ext/js/data/database.js b/ext/js/data/database.js index a53c8ddb45..2587382bdd 100644 --- a/ext/js/data/database.js +++ b/ext/js/data/database.js @@ -30,7 +30,7 @@ export class Database { /** * @param {string} databaseName * @param {number} version - * @param {import('database').StructureDefinition[]} structure + * @param {import('database').StructureDefinition[]?} structure */ async open(databaseName, version, structure) { if (this._db !== null) { @@ -43,7 +43,9 @@ export class Database { try { this._isOpening = true; this._db = await this._open(databaseName, version, (db, transaction, oldVersion) => { - this._upgrade(db, transaction, oldVersion, structure); + if (structure !== null) { + this._upgrade(db, transaction, oldVersion, structure); + } }); } finally { this._isOpening = false; diff --git a/ext/js/dictionary/dictionary-database-worker-handler.js b/ext/js/dictionary/dictionary-database-worker-handler.js new file mode 100644 index 0000000000..625a3123c0 --- /dev/null +++ b/ext/js/dictionary/dictionary-database-worker-handler.js @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2023-2024 Yomitan Authors + * Copyright (C) 2021-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {log} from '../core/log.js'; +import {DictionaryDatabase} from './dictionary-database.js'; + +export class DictionaryDatabaseWorkerHandler { + constructor() { + /** @type {DictionaryDatabase?} */ + this._dictionaryDatabase = null; + } + + /** + * + */ + async prepare() { + this._dictionaryDatabase = new DictionaryDatabase(); + try { + await this._dictionaryDatabase.prepare(); + } catch (e) { + log.error(e); + } + self.addEventListener('message', this._onMessage.bind(this), false); + } + // Private + + /** + * @param {MessageEvent} event + */ + _onMessage(event) { + const {action} = event.data; + switch (action) { + case 'connectToDatabaseWorker': + void this._dictionaryDatabase?.connectToDatabaseWorker(event.ports[0]); + break; + } + } +} diff --git a/ext/js/dictionary/dictionary-database-worker-main.js b/ext/js/dictionary/dictionary-database-worker-main.js new file mode 100644 index 0000000000..906aa2ce8a --- /dev/null +++ b/ext/js/dictionary/dictionary-database-worker-main.js @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023-2024 Yomitan Authors + * Copyright (C) 2021-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {log} from '../core/log.js'; +import {DictionaryDatabaseWorkerHandler} from './dictionary-database-worker-handler.js'; + +/** Entry point. */ +function main() { + try { + const dictionaryDatabaseWorkerHandler = new DictionaryDatabaseWorkerHandler(); + void dictionaryDatabaseWorkerHandler.prepare(); + } catch (e) { + log.error(e); + } +} + +main(); diff --git a/ext/js/dictionary/dictionary-database.js b/ext/js/dictionary/dictionary-database.js index 4eb601e8aa..98347de52c 100644 --- a/ext/js/dictionary/dictionary-database.js +++ b/ext/js/dictionary/dictionary-database.js @@ -16,6 +16,8 @@ * along with this program. If not, see . */ +import {initWasm, Resvg} from '../../lib/resvg-wasm.js'; +import {createApiMap, invokeApiMapHandler} from '../core/api-map.js'; import {log} from '../core/log.js'; import {safePerformance} from '../core/safe-performance.js'; import {stringReverse} from '../core/utilities.js'; @@ -35,6 +37,8 @@ export class DictionaryDatabase { this._createOnlyQuery3 = (item) => IDBKeyRange.only(item.term); /** @type {import('dictionary-database').CreateQuery} */ this._createOnlyQuery4 = (item) => IDBKeyRange.only(item.path); + /** @type {import('dictionary-database').CreateQuery} */ + this._createOnlyQuery5 = (item) => IDBKeyRange.only(item.path); /** @type {import('dictionary-database').CreateQuery} */ this._createBoundQuery1 = (item) => IDBKeyRange.bound(item, `${item}\uffff`, false, false); /** @type {import('dictionary-database').CreateQuery} */ @@ -54,14 +58,33 @@ export class DictionaryDatabase { this._createKanjiMetaBind = this._createKanjiMeta.bind(this); /** @type {import('dictionary-database').CreateResult} */ this._createMediaBind = this._createMedia.bind(this); + /** @type {import('dictionary-database').CreateResult} */ + this._createDrawMediaBind = this._createDrawMedia.bind(this); + + /** + * @type {Worker?} + */ + this._worker = null; + + /** + * @type {Uint8Array?} + */ + this._resvgFontBuffer = null; + + /** @type {import('dictionary-database').ApiMap} */ + this._apiMap = createApiMap([ + ['drawMedia', this._onDrawMedia.bind(this)], + ]); } - /** */ + /** + * do upgrades for the IndexedDB schema (basically limited to adding new stores when needed) + */ async prepare() { - await this._db.open( - this._dbName, - 60, - /** @type {import('database').StructureDefinition[]} */ + // do not do upgrades in web workers as they are considered to be children of the main thread and are not responsible for database upgrades + const isWorker = self.constructor.name !== 'Window'; + const upgrade = + /** @type {import('database').StructureDefinition[]?} */ ([ /** @type {import('database').StructureDefinition} */ ({ @@ -129,8 +152,30 @@ export class DictionaryDatabase { }, }, }, - ]), + ]); + await this._db.open( + this._dbName, + 60, + isWorker ? null : upgrade, ); + + // when we are not a worker ourselves, create a worker which is basically just a wrapper around this class, which we can use to offload some functions to + if (!isWorker) { + this._worker = new Worker('/js/dictionary/dictionary-database-worker-main.js', {type: 'module'}); + this._worker.addEventListener('error', (event) => { + log.log('Worker terminated with error:', event); + }); + this._worker.addEventListener('unhandledrejection', (event) => { + log.log('Unhandled promise rejection in worker:', event); + }); + } else { + // when we are the worker, prepare to need to do some SVG work and load appropriate wasm & fonts + await initWasm(fetch('/lib/resvg.wasm')); + + const font = await fetch('/fonts/NotoSansJP-Regular.ttf'); + const fontData = await font.arrayBuffer(); + this._resvgFontBuffer = new Uint8Array(fontData); + } } /** */ @@ -347,6 +392,98 @@ export class DictionaryDatabase { return this._findMultiBulk('media', ['path'], items, this._createOnlyQuery4, predicate, this._createMediaBind); } + /** + * @param {import('dictionary-database').DrawMediaRequest[]} items + * @param {MessagePort} source + */ + async drawMedia(items, source) { + if (this._worker !== null) { // if a worker is available, offload the work to it + this._worker.postMessage({action: 'drawMedia', params: {items}}, [source]); + return; + } + // otherwise, you are the worker, so do the work + safePerformance.mark('drawMedia:start'); + + // merge items with the same path to reduce the number of database queries. collects the canvases into a single array for each path. + /** @type {Map} */ + const groupedItems = new Map(); + for (const item of items) { + const {path, dictionary, canvasIndex, canvasWidth, canvasHeight, generation} = item; + const key = `${path}:::${dictionary}`; + if (!groupedItems.has(key)) { + groupedItems.set(key, {path, dictionary, canvasIndexes: [], canvasWidth, canvasHeight, generation}); + } + groupedItems.get(key)?.canvasIndexes.push(canvasIndex); + } + const groupedItemsArray = [...groupedItems.values()]; + + /** @type {import('dictionary-database').FindPredicate} */ + const predicate = (row, item) => (row.dictionary === item.dictionary); + const results = await this._findMultiBulk('media', ['path'], groupedItemsArray, this._createOnlyQuery5, predicate, this._createDrawMediaBind); + + // move all svgs to front to have a hotter loop + results.sort((a, _b) => (a.mediaType === 'image/svg+xml' ? -1 : 1)); + + safePerformance.mark('drawMedia:draw:start'); + for (const m of results) { + if (m.mediaType === 'image/svg+xml') { + safePerformance.mark('drawMedia:draw:svg:start'); + /** @type {import('@resvg/resvg-wasm').ResvgRenderOptions} */ + const opts = { + fitTo: { + mode: 'width', + value: m.canvasWidth, + }, + font: { + fontBuffers: this._resvgFontBuffer !== null ? [this._resvgFontBuffer] : [], + }, + }; + const resvgJS = new Resvg(new Uint8Array(m.content), opts); + const render = resvgJS.render(); + source.postMessage({action: 'drawBufferToCanvases', params: {buffer: render.pixels.buffer, width: render.width, height: render.height, canvasIndexes: m.canvasIndexes, generation: m.generation}}, [render.pixels.buffer]); + safePerformance.mark('drawMedia:draw:svg:end'); + safePerformance.measure('drawMedia:draw:svg', 'drawMedia:draw:svg:start', 'drawMedia:draw:svg:end'); + } else { + safePerformance.mark('drawMedia:draw:raster:start'); + + // ImageDecoder is slightly faster than Blob/createImageBitmap, but + // 1) it is not available in Firefox <133 + // 2) it is available in Firefox >=133, but it's not possible to transfer VideoFrames cross-process + // + // So the second branch is a fallback for all versions of Firefox and doesn't use ImageDecoder at all + // The second branch can eventually be changed to use ImageDecoder when we are okay with dropping support for Firefox <133 + // The branches can be unified entirely when Firefox implements support for transferring VideoFrames cross-process in postMessage + if ('serviceWorker' in navigator) { // this is just a check for chrome, we don't actually use service worker functionality here + // eslint-disable-next-line no-undef + const imageDecoder = new ImageDecoder({type: m.mediaType, data: m.content}); + await imageDecoder.decode().then((decodedImageResult) => { + source.postMessage({action: 'drawDecodedImageToCanvases', params: {decodedImage: decodedImageResult.image, canvasIndexes: m.canvasIndexes, generation: m.generation}}, [decodedImageResult.image]); + }); + } else { + const image = new Blob([m.content], {type: m.mediaType}); + // eslint-disable-next-line no-undef + await createImageBitmap(image).then((decodedImage) => { + // we need to do a dumb hack where we convert this ImageBitmap to an ImageData by drawing it to a temporary canvas, because Firefox doesn't support transferring ImageBitmaps cross-process + const canvas = new OffscreenCanvas(decodedImage.width, decodedImage.height); + const ctx = canvas.getContext('2d'); + if (ctx !== null) { + ctx.drawImage(decodedImage, 0, 0); + const imageData = ctx.getImageData(0, 0, decodedImage.width, decodedImage.height); + source.postMessage({action: 'drawBufferToCanvases', params: {buffer: imageData.data.buffer, width: decodedImage.width, height: decodedImage.height, canvasIndexes: m.canvasIndexes, generation: m.generation}}, [imageData.data.buffer]); + } + }); + } + safePerformance.mark('drawMedia:draw:raster:end'); + safePerformance.measure('drawMedia:draw:raster', 'drawMedia:draw:raster:start', 'drawMedia:draw:raster:end'); + } + } + safePerformance.mark('drawMedia:draw:end'); + safePerformance.measure('drawMedia:draw', 'drawMedia:draw:start', 'drawMedia:draw:end'); + + safePerformance.mark('drawMedia:end'); + safePerformance.measure('drawMedia', 'drawMedia:start', 'drawMedia:end'); + } + /** * @returns {Promise} */ @@ -478,10 +615,14 @@ export class DictionaryDatabase { let completeCount = 0; const requiredCompleteCount = itemCount * indexCount; /** - * @param {TRow[]} rows - * @param {import('dictionary-database').FindMultiBulkData} data + * @param {TItem} item + * @returns {(rows: TRow[], data: import('dictionary-database').FindMultiBulkData) => void} */ - const onGetAll = (rows, data) => { + const onGetAll = (item) => (rows, data) => { + if (typeof item === 'object' && item !== null && 'path' in item) { + safePerformance.mark(`findMultiBulk:onGetAll:${item.path}:end`); + safePerformance.measure(`findMultiBulk:onGetAll:${item.path}`, `findMultiBulk:onGetAll:${item.path}:start`, `findMultiBulk:onGetAll:${item.path}:end`); + } for (const row of rows) { if (predicate(row, data.item)) { results.push(createResult(row, data)); @@ -489,17 +630,25 @@ export class DictionaryDatabase { } if (++completeCount >= requiredCompleteCount) { resolve(results); + safePerformance.mark('findMultiBulk:end'); + safePerformance.measure('findMultiBulk', 'findMultiBulk:start', 'findMultiBulk:end'); } }; + safePerformance.mark('findMultiBulk:getAll:start'); for (let i = 0; i < itemCount; ++i) { const item = items[i]; const query = createQuery(item); for (let j = 0; j < indexCount; ++j) { /** @type {import('dictionary-database').FindMultiBulkData} */ const data = {item, itemIndex: i, indexIndex: j}; - this._db.getAll(indexList[j], query, onGetAll, reject, data); + if (typeof item === 'object' && item !== null && 'path' in item) { + safePerformance.mark(`findMultiBulk:onGetAll:${item.path}:start`); + } + this._db.getAll(indexList[j], query, onGetAll(item), reject, data); } } + safePerformance.mark('findMultiBulk:getAll:end'); + safePerformance.measure('findMultiBulk:getAll', 'findMultiBulk:getAll:start', 'findMultiBulk:getAll:end'); }); } @@ -661,6 +810,16 @@ export class DictionaryDatabase { return {index, dictionary, path, mediaType, width, height, content}; } + /** + * @param {import('dictionary-database').MediaDataArrayBufferContent} row + * @param {import('dictionary-database').FindMultiBulkData} data + * @returns {import('dictionary-database').DrawMedia} + */ + _createDrawMedia(row, {itemIndex: index, item: {canvasIndexes, canvasWidth, generation}}) { + const {dictionary, path, mediaType, width, height, content} = row; + return {index, dictionary, path, mediaType, width, height, content, canvasIndexes, canvasWidth, generation}; + } + /** * @param {unknown} field * @returns {string[]} @@ -668,4 +827,27 @@ export class DictionaryDatabase { _splitField(field) { return typeof field === 'string' && field.length > 0 ? field.split(' ') : []; } + + // Parent-Worker API + + /** + * @param {MessagePort} port + */ + async connectToDatabaseWorker(port) { + if (this._worker !== null) { + // executes outside of worker + this._worker.postMessage({action: 'connectToDatabaseWorker'}, [port]); + return; + } + // executes inside worker + port.onmessage = (/** @type {MessageEvent} */event) => { + const {action, params} = event.data; + return invokeApiMapHandler(this._apiMap, action, params, [port], () => {}); + }; + } + + /** @type {import('dictionary-database').ApiHandler<'drawMedia'>} */ + _onDrawMedia(params, port) { + void this.drawMedia(params.requests, port); + } } diff --git a/ext/js/display/display-content-manager.js b/ext/js/display/display-content-manager.js index 070ab81ab6..09693f6ffb 100644 --- a/ext/js/display/display-content-manager.js +++ b/ext/js/display/display-content-manager.js @@ -17,7 +17,6 @@ */ import {EventListenerCollection} from '../core/event-listener-collection.js'; -import {base64ToArrayBuffer} from '../data/array-buffer-util.js'; /** * The content manager which is used when generating HTML display content. @@ -32,47 +31,36 @@ export class DisplayContentManager { this._display = display; /** @type {import('core').TokenObject} */ this._token = {}; - /** @type {Map>>} */ - this._mediaCache = new Map(); - /** @type {import('display-content-manager').LoadMediaDataInfo[]} */ - this._loadMediaData = []; /** @type {EventListenerCollection} */ this._eventListeners = new EventListenerCollection(); + /** @type {import('display-content-manager').LoadMediaRequest[]} */ + this._loadMediaRequests = []; + } + + /** @type {import('display-content-manager').LoadMediaRequest[]} */ + get loadMediaRequests() { + return this._loadMediaRequests; } /** - * Attempts to load the media file from a given dictionary. - * @param {string} path The path to the media file in the dictionary. - * @param {string} dictionary The name of the dictionary. - * @param {import('display-content-manager').OnLoadCallback} onLoad The callback that is executed if the media was loaded successfully. - * No assumptions should be made about the synchronicity of this callback. - * @param {import('display-content-manager').OnUnloadCallback} onUnload The callback that is executed when the media should be unloaded. + * Queues loading media file from a given dictionary. + * @param {string} path + * @param {string} dictionary + * @param {OffscreenCanvas} canvas */ - loadMedia(path, dictionary, onLoad, onUnload) { - void this._loadMedia(path, dictionary, onLoad, onUnload); + loadMedia(path, dictionary, canvas) { + this._loadMediaRequests.push({path, dictionary, canvas}); } /** * Unloads all media that has been loaded. */ unloadAll() { - for (const {onUnload, loaded} of this._loadMediaData) { - if (typeof onUnload === 'function') { - onUnload(loaded); - } - } - this._loadMediaData = []; - - for (const map of this._mediaCache.values()) { - for (const result of map.values()) { - void this._revokeUrl(result); - } - } - this._mediaCache.clear(); - this._token = {}; this._eventListeners.removeAllEventListeners(); + + this._loadMediaRequests = []; } /** @@ -91,63 +79,11 @@ export class DisplayContentManager { } /** - * @param {string} path - * @param {string} dictionary - * @param {import('display-content-manager').OnLoadCallback} onLoad - * @param {import('display-content-manager').OnUnloadCallback} onUnload + * Execute media requests */ - async _loadMedia(path, dictionary, onLoad, onUnload) { - const token = this._token; - const media = await this._getMedia(path, dictionary); - if (token !== this._token || media === null) { return; } - - /** @type {import('display-content-manager').LoadMediaDataInfo} */ - const data = {onUnload, loaded: false}; - this._loadMediaData.push(data); - onLoad(media.url); - data.loaded = true; - } - - /** - * @param {string} path - * @param {string} dictionary - * @returns {Promise} - */ - _getMedia(path, dictionary) { - /** @type {Promise|undefined} */ - let promise; - let dictionaryCache = this._mediaCache.get(dictionary); - if (typeof dictionaryCache !== 'undefined') { - promise = dictionaryCache.get(path); - } else { - dictionaryCache = new Map(); - this._mediaCache.set(dictionary, dictionaryCache); - } - - if (typeof promise === 'undefined') { - promise = this._getMediaData(path, dictionary); - dictionaryCache.set(path, promise); - } - - return promise; - } - - /** - * @param {string} path - * @param {string} dictionary - * @returns {Promise} - */ - async _getMediaData(path, dictionary) { - const token = this._token; - const datas = await this._display.application.api.getMedia([{path, dictionary}]); - if (token === this._token && datas.length > 0) { - const data = datas[0]; - const buffer = base64ToArrayBuffer(data.content); - const blob = new Blob([buffer], {type: data.mediaType}); - const url = URL.createObjectURL(blob); - return {data, url}; - } - return null; + async executeMediaRequests() { + this._display.application.api.drawMedia(this._loadMediaRequests, this._loadMediaRequests.map(({canvas}) => canvas)); + this._loadMediaRequests = []; } /** @@ -177,13 +113,4 @@ export class DisplayContentManager { content: null, }); } - - /** - * @param {Promise} data - */ - async _revokeUrl(data) { - const result = await data; - if (result === null) { return; } - URL.revokeObjectURL(result.url); - } } diff --git a/ext/js/display/display-generator.js b/ext/js/display/display-generator.js index 48d0c797c8..af56af02ce 100644 --- a/ext/js/display/display-generator.js +++ b/ext/js/display/display-generator.js @@ -43,6 +43,13 @@ export class DisplayGenerator { this._language = 'ja'; } + /** @type {import('./display-content-manager.js').DisplayContentManager} */ + get contentManager() { return this._contentManager; } + + set contentManager(contentManager) { + this._contentManager = contentManager; + } + /** */ async prepare() { await this._templates.loadFromFiles(['/templates-display.html']); diff --git a/ext/js/display/display.js b/ext/js/display/display.js index 4066e33196..0b4bf6b0fc 100644 --- a/ext/js/display/display.js +++ b/ext/js/display/display.js @@ -795,19 +795,18 @@ export class Display extends EventDispatcher { this._setContentToken = token; try { // Clear - safePerformance.mark('display:clear:start'); + safePerformance.mark('display:_onStateChanged:clear:start'); this._closePopups(); this._closeAllPopupMenus(); this._eventListeners.removeAllEventListeners(); - this._contentManager.unloadAll(); this._hideTagNotification(false); this._hideInflectionNotification(false); this._triggerContentClear(); this._dictionaryEntries = []; this._dictionaryEntryNodes = []; this._elementOverflowController.clearElements(); - safePerformance.mark('display:clear:end'); - safePerformance.measure('display:clear', 'display:clear:start', 'display:clear:end'); + safePerformance.mark('display:_onStateChanged:clear:end'); + safePerformance.measure('display:_onStateChanged:clear', 'display:_onStateChanged:clear:start', 'display:_onStateChanged:clear:end'); // Prepare safePerformance.mark('display:_onStateChanged:prepare:start'); @@ -1417,7 +1416,8 @@ export class Display extends EventDispatcher { safePerformance.mark('display:contentUpdate:start'); this._triggerContentUpdateStart(); - for (let i = 0, ii = dictionaryEntries.length; i < ii; ++i) { + let i = 0; + for (const dictionaryEntry of dictionaryEntries) { safePerformance.mark('display:createEntry:start'); if (i > 0) { @@ -1425,7 +1425,8 @@ export class Display extends EventDispatcher { if (this._setContentToken !== token) { return; } } - const dictionaryEntry = dictionaryEntries[i]; + safePerformance.mark('display:createEntryReal:start'); + const entry = ( dictionaryEntry.type === 'term' ? this._displayGenerator.createTermEntry(dictionaryEntry, this._dictionaryInfo) : @@ -1435,16 +1436,31 @@ export class Display extends EventDispatcher { this._dictionaryEntryNodes.push(entry); this._addEntryEventListeners(entry); this._triggerContentUpdateEntry(dictionaryEntry, entry, i); + safePerformance.mark('display:waitMedia:start'); + safePerformance.mark('display:waitMedia:end'); + safePerformance.measure('display:waitMedia', 'display:waitMedia:start', 'display:waitMedia:end'); + if (this._setContentToken !== token) { return; } container.appendChild(entry); + if (focusEntry === i) { this._focusEntry(i, 0, false); } this._elementOverflowController.addElements(entry); + safePerformance.mark('display:createEntryReal:end'); + safePerformance.measure('display:createEntryReal', 'display:createEntryReal:start', 'display:createEntryReal:end'); + safePerformance.mark('display:createEntry:end'); safePerformance.measure('display:createEntry', 'display:createEntry:start', 'display:createEntry:end'); + + if (i === 0) { + void this._contentManager.executeMediaRequests(); // prioritize loading media for first entry since it is visible + } + ++i; } + if (this._setContentToken !== token) { return; } + void this._contentManager.executeMediaRequests(); if (typeof scrollX === 'number' || typeof scrollY === 'number') { let {x, y} = this._windowScroll; diff --git a/ext/js/display/media-drawing-worker.js b/ext/js/display/media-drawing-worker.js new file mode 100644 index 0000000000..6138e5ecc1 --- /dev/null +++ b/ext/js/display/media-drawing-worker.js @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {API} from '../comm/api.js'; +import {createApiMap, invokeApiMapHandler} from '../core/api-map.js'; +import {log} from '../core/log.js'; +import {WebExtension} from '../extension/web-extension.js'; + +export class MediaDrawingWorker { + constructor() { + /** @type {number} */ + this._generation = 0; + + /** @type {MessagePort?} */ + this._dbPort = null; + + /** @type {import('api').PmApiMap} */ + this._fromApplicationApiMap = createApiMap([ + ['drawMedia', this._onDrawMedia.bind(this)], + ['connectToDatabaseWorker', this._onConnectToDatabaseWorker.bind(this)], + ]); + + /** @type {import('api').PmApiMap} */ + this._fromDatabaseApiMap = createApiMap([ + ['drawBufferToCanvases', this._onDrawBufferToCanvases.bind(this)], + ['drawDecodedImageToCanvases', this._onDrawDecodedImageToCanvases.bind(this)], + ]); + + /** @type {Map} */ + this._canvasesByGeneration = new Map(); + + /** + * @type {API} + */ + this._api = new API(new WebExtension()); + } + + /** + * + */ + async prepare() { + addEventListener('message', (event) => { + /** @type {import('api').PmApiMessageAny} */ + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const message = event.data; + return invokeApiMapHandler(this._fromApplicationApiMap, message.action, message.params, [event.ports], () => {}); + }); + } + + /** @type {import('api').PmApiHandler<'drawMedia'>} */ + async _onDrawMedia({requests}) { + this._generation++; + this._canvasesByGeneration.set(this._generation, requests.map((request) => request.canvas)); + this._cleanOldGenerations(); + const newRequests = requests.map((request, index) => ({...request, canvas: null, generation: this._generation, canvasIndex: index, canvasWidth: request.canvas.width, canvasHeight: request.canvas.height})); + if (this._dbPort !== null) { + this._dbPort.postMessage({action: 'drawMedia', params: {requests: newRequests}}); + } else { + log.error('no database port available'); + } + } + + /** @type {import('api').PmApiHandler<'drawBufferToCanvases'>} */ + async _onDrawBufferToCanvases({buffer, width, height, canvasIndexes, generation}) { + try { + const canvases = this._canvasesByGeneration.get(generation); + if (typeof canvases === 'undefined') { + return; + } + const imageData = new ImageData(new Uint8ClampedArray(buffer), width, height); + for (const ci of canvasIndexes) { + const c = canvases[ci]; + c.getContext('2d')?.putImageData(imageData, 0, 0); + } + } catch (e) { + log.error(e); + } + } + + /** @type {import('api').PmApiHandler<'drawDecodedImageToCanvases'>} */ + async _onDrawDecodedImageToCanvases({decodedImage, canvasIndexes, generation}) { + try { + const canvases = this._canvasesByGeneration.get(generation); + if (typeof canvases === 'undefined') { + return; + } + for (const ci of canvasIndexes) { + const c = canvases[ci]; + c.getContext('2d')?.drawImage(decodedImage, 0, 0, c.width, c.height); + } + } catch (e) { + log.error(e); + } + } + + /** @type {import('api').PmApiHandler<'connectToDatabaseWorker'>} */ + async _onConnectToDatabaseWorker(_params, ports) { + if (ports === null) { + return; + } + const dbPort = ports[0]; + this._dbPort = dbPort; + dbPort.addEventListener('message', (/** @type {MessageEvent} */ event) => { + const message = event.data; + return invokeApiMapHandler(this._fromDatabaseApiMap, message.action, message.params, [event.ports], () => {}); + }); + dbPort.start(); + } + + /** + * @param {number} keepNGenerations Number of generations to keep, defaults to 2 (the current generation and the one before it). + */ + _cleanOldGenerations(keepNGenerations = 2) { + const generations = [...this._canvasesByGeneration.keys()]; + for (const g of generations) { + if (g <= this._generation - keepNGenerations) { + this._canvasesByGeneration.delete(g); + } + } + } +} + +const mediaDrawingWorker = new MediaDrawingWorker(); +await mediaDrawingWorker.prepare(); diff --git a/ext/js/display/structured-content-generator.js b/ext/js/display/structured-content-generator.js index e42180df64..add061f49b 100644 --- a/ext/js/display/structured-content-generator.js +++ b/ext/js/display/structured-content-generator.js @@ -16,7 +16,9 @@ * along with this program. If not, see . */ +import {DisplayContentManager} from '../display/display-content-manager.js'; import {getLanguageFromText} from '../language/text-utilities.js'; +import {AnkiTemplateRendererContentManager} from '../templates/anki-template-renderer-content-manager.js'; export class StructuredContentGenerator { /** @@ -64,7 +66,6 @@ export class StructuredContentGenerator { preferredWidth, preferredHeight, title, - alt, pixelated, imageRendering, appearance, @@ -97,23 +98,6 @@ export class StructuredContentGenerator { const imageContainer = this._createElement('span', 'gloss-image-container'); node.appendChild(imageContainer); - const aspectRatioSizer = this._createElement('span', 'gloss-image-sizer'); - imageContainer.appendChild(aspectRatioSizer); - - const imageBackground = this._createElement('span', 'gloss-image-background'); - imageContainer.appendChild(imageBackground); - - const image = /** @type {HTMLImageElement} */ (this._createElement('img', 'gloss-image')); - image.alt = typeof alt === 'string' ? alt : ''; - imageContainer.appendChild(image); - - const overlay = this._createElement('span', 'gloss-image-container-overlay'); - imageContainer.appendChild(overlay); - - const linkText = this._createElement('span', 'gloss-image-link-text'); - linkText.textContent = 'Image'; - node.appendChild(linkText); - node.dataset.path = path; node.dataset.dictionary = dictionary; node.dataset.imageLoadState = 'not-loaded'; @@ -130,22 +114,45 @@ export class StructuredContentGenerator { node.dataset.sizeUnits = sizeUnits; } - imageContainer.style.width = `${usedWidth}em`; if (typeof border === 'string') { imageContainer.style.border = border; } if (typeof borderRadius === 'string') { imageContainer.style.borderRadius = borderRadius; } if (typeof title === 'string') { imageContainer.title = title; } - aspectRatioSizer.style.paddingTop = `${invAspectRatio * 100}%`; - if (this._contentManager !== null) { - this._contentManager.loadMedia( - path, - dictionary, - (url) => this._setImageData(node, image, imageBackground, url, false), - () => this._setImageData(node, image, imageBackground, null, true), - ); + const image = this._contentManager instanceof DisplayContentManager ? + /** @type {HTMLCanvasElement} */ (this._createElement('canvas', 'gloss-image')) : + /** @type {HTMLImageElement} */ (this._createElement('img', 'gloss-image')); + if (sizeUnits === 'em' && (hasPreferredWidth || hasPreferredHeight)) { + image.style.width = `${usedWidth}em`; + image.style.height = `${usedWidth * invAspectRatio}em`; + image.width = usedWidth * 14 * 2 * window.devicePixelRatio; + image.height = usedWidth * invAspectRatio * 14 * 2 * window.devicePixelRatio; + } else { + image.width = usedWidth; + image.height = usedWidth * invAspectRatio; + } + imageContainer.appendChild(image); + + if (this._contentManager instanceof DisplayContentManager) { + this._contentManager.loadMedia( + path, + dictionary, + (/** @type {HTMLCanvasElement} */(image)).transferControlToOffscreen(), + ); + } else if (this._contentManager instanceof AnkiTemplateRendererContentManager) { + this._contentManager.loadMedia( + path, + dictionary, + (url) => { + (/** @type {HTMLImageElement} */(image)).src = url; + }, + () => { + (/** @type {HTMLImageElement} */(image)).removeAttribute('src'); + }, + ); + } } return node; @@ -224,27 +231,6 @@ export class StructuredContentGenerator { } } - /** - * @param {HTMLAnchorElement} node - * @param {HTMLImageElement} image - * @param {HTMLElement} imageBackground - * @param {?string} url - * @param {boolean} unloaded - */ - _setImageData(node, image, imageBackground, url, unloaded) { - if (url !== null) { - image.src = url; - node.href = url; - node.dataset.imageLoadState = 'loaded'; - imageBackground.style.setProperty('--image', `url("${url}")`); - } else { - image.removeAttribute('src'); - node.removeAttribute('href'); - node.dataset.imageLoadState = unloaded ? 'unloaded' : 'load-error'; - imageBackground.style.removeProperty('--image'); - } - } - /** * @param {import('structured-content').Element} content * @param {string} dictionary diff --git a/jsconfig.json b/jsconfig.json index 6981b5f40b..4057557f02 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -17,6 +17,7 @@ }, "types": [ "chrome", + "dom-webcodecs", "firefox-webext-browser", "handlebars", "jszip", @@ -25,6 +26,12 @@ "zip.js", "dexie", "ajv" + ], + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable", + "WebWorker" ] }, "include": [ diff --git a/package-lock.json b/package-lock.json index ed35a4da13..e6b2bf0ddf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "license": "GPL-3.0-or-later", "dependencies": { + "@resvg/resvg-wasm": "^2.6.2", "@zip.js/zip.js": "^2.7.45", "dexie": "^3.2.5", "dexie-export-import": "^4.1.2", @@ -26,6 +27,7 @@ "@types/browserify": "^12.0.40", "@types/chrome": "^0.0.268", "@types/css": "^0.0.37", + "@types/dom-webcodecs": "^0.1.13", "@types/events": "^3.0.3", "@types/firefox-webext-browser": "^120.0.3", "@types/jsdom": "^21.1.6", @@ -1231,6 +1233,14 @@ "node": ">=16" } }, + "node_modules/@resvg/resvg-wasm": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-wasm/-/resvg-wasm-2.6.2.tgz", + "integrity": "sha512-FqALmHI8D4o6lk/LRWDnhw95z5eO+eAa6ORjVg09YRR7BkcM6oPHU9uyC0gtQG5vpFLvgpeU4+zEAz2H8APHNw==", + "engines": { + "node": ">= 10" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.24.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.0.tgz", @@ -1658,6 +1668,12 @@ "integrity": "sha512-IVhWCNH1mw3VRjkOMHsxVAcnANhee9w//TX1fqmALP628Dzf6VMG1LRnOngpptnrilcWCkmcY1tj6QkKGUy0CA==", "dev": true }, + "node_modules/@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "dev": true + }, "node_modules/@types/eslint": { "version": "8.56.5", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.5.tgz", @@ -10937,6 +10953,11 @@ "playwright": "1.44.1" } }, + "@resvg/resvg-wasm": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-wasm/-/resvg-wasm-2.6.2.tgz", + "integrity": "sha512-FqALmHI8D4o6lk/LRWDnhw95z5eO+eAa6ORjVg09YRR7BkcM6oPHU9uyC0gtQG5vpFLvgpeU4+zEAz2H8APHNw==" + }, "@rollup/rollup-android-arm-eabi": { "version": "4.24.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.0.tgz", @@ -11216,6 +11237,12 @@ "integrity": "sha512-IVhWCNH1mw3VRjkOMHsxVAcnANhee9w//TX1fqmALP628Dzf6VMG1LRnOngpptnrilcWCkmcY1tj6QkKGUy0CA==", "dev": true }, + "@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "dev": true + }, "@types/eslint": { "version": "8.56.5", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.5.tgz", diff --git a/package.json b/package.json index 6bdd980446..6586a8717f 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@types/browserify": "^12.0.40", "@types/chrome": "^0.0.268", "@types/css": "^0.0.37", + "@types/dom-webcodecs": "^0.1.13", "@types/events": "^3.0.3", "@types/firefox-webext-browser": "^120.0.3", "@types/jsdom": "^21.1.6", @@ -106,6 +107,7 @@ "vitest": "^1.2.2" }, "dependencies": { + "@resvg/resvg-wasm": "^2.6.2", "@zip.js/zip.js": "^2.7.45", "dexie": "^3.2.5", "dexie-export-import": "^4.1.2", diff --git a/test/data/anki-note-builder-test-results.json b/test/data/anki-note-builder-test-results.json index b496d0b880..617464d3ee 100644 --- a/test/data/anki-note-builder-test-results.json +++ b/test/data/anki-note-builder-test-results.json @@ -863,12 +863,12 @@ "frequency-average-occurrence": "0", "furigana": "画像がぞう", "furigana-plain": "画像[がぞう]", - "glossary": "
(n, termsDictAlias)
", - "glossary-brief": "
", - "glossary-no-dictionary": "
(n)
", - "glossary-first": "
(n, termsDictAlias)
", - "glossary-first-brief": "
", - "glossary-first-no-dictionary": "
(n)
", + "glossary": "
(n, termsDictAlias)
  • gazou definition 1
", + "glossary-brief": "
  • gazou definition 1
", + "glossary-no-dictionary": "
(n)
  • gazou definition 1
", + "glossary-first": "
(n, termsDictAlias)
  • gazou definition 1
", + "glossary-first-brief": "
  • gazou definition 1
", + "glossary-first-no-dictionary": "
(n)
  • gazou definition 1
", "part-of-speech": "Noun", "pitch-accents": "", "pitch-accent-graphs": "", @@ -1570,12 +1570,12 @@ "frequency-average-occurrence": "0", "furigana": "画像がぞう", "furigana-plain": "画像[がぞう]", - "glossary": "
(n, termsDictAlias)
", - "glossary-brief": "
", - "glossary-no-dictionary": "
(n)
", - "glossary-first": "
(n, termsDictAlias)
", - "glossary-first-brief": "
", - "glossary-first-no-dictionary": "
(n)
", + "glossary": "
(n, termsDictAlias)
  • gazou definition 1
", + "glossary-brief": "
  • gazou definition 1
", + "glossary-no-dictionary": "
(n)
  • gazou definition 1
", + "glossary-first": "
(n, termsDictAlias)
  • gazou definition 1
", + "glossary-first-brief": "
  • gazou definition 1
", + "glossary-first-no-dictionary": "
(n)
  • gazou definition 1
", "part-of-speech": "Noun", "pitch-accents": "", "pitch-accent-graphs": "", diff --git a/test/data/translator-test-results.json b/test/data/translator-test-results.json index 1990469009..5fa0f31845 100644 --- a/test/data/translator-test-results.json +++ b/test/data/translator-test-results.json @@ -10869,7 +10869,6 @@ "dictionary": "Test Dictionary 2", "dictionaryIndex": 0, "dictionaryAlias": "termsDictAlias", - "hasReading": true, "frequency": 10, "displayValue": null, @@ -19650,7 +19649,7 @@ ] }, { - "name": "Find term using primary reading 1", + "name": "Find terms using primary reading 1", "originalTextLength": 2, "dictionaryEntries": [ { @@ -19814,10 +19813,9 @@ ] }, { - "name": "Find term using primary reading 2", + "name": "Find terms using primary reading 2", "originalTextLength": 2, "dictionaryEntries": [ - { "type": "term", "isPrimary": true, diff --git a/test/database.test.js b/test/database.test.js index 8bc95ceb3c..50fc213449 100644 --- a/test/database.test.js +++ b/test/database.test.js @@ -26,9 +26,11 @@ import {parseJson} from '../dev/json.js'; import {DictionaryDatabase} from '../ext/js/dictionary/dictionary-database.js'; import {DictionaryImporter} from '../ext/js/dictionary/dictionary-importer.js'; import {DictionaryImporterMediaLoader} from './mocks/dictionary-importer-media-loader.js'; +import {setupStubs} from './utilities/database.js'; const dirname = pathDirname(fileURLToPath(import.meta.url)); +setupStubs(); vi.stubGlobal('IDBKeyRange', IDBKeyRange); /** diff --git a/test/dictionary-data.test.js b/test/dictionary-data.test.js index cfb951d822..c3e9bb8171 100644 --- a/test/dictionary-data.test.js +++ b/test/dictionary-data.test.js @@ -22,13 +22,17 @@ import {describe} from 'vitest'; import {parseJson} from '../dev/json.js'; import {createTranslatorTest} from './fixtures/translator-test.js'; import {createTestAnkiNoteData, getTemplateRenderResults} from './utilities/anki.js'; +import {setupStubs} from './utilities/database.js'; import {createFindKanjiOptions, createFindTermsOptions} from './utilities/translator.js'; +setupStubs(); + const dirname = path.dirname(fileURLToPath(import.meta.url)); const dictionaryName = 'Test Dictionary 2'; const test = await createTranslatorTest(void 0, path.join(dirname, 'data/dictionaries/valid-dictionary1'), dictionaryName); describe('Dictionary data', () => { + console.log('test'); const testInputsFilePath = path.join(dirname, 'data/translator-test-inputs.json'); /** @type {import('test/translator').TranslatorTestInputs} */ const {optionsPresets, tests} = parseJson(readFileSync(testInputsFilePath, {encoding: 'utf8'})); diff --git a/test/dictionary-data.write.js b/test/dictionary-data.write.js index 880d23798a..0dc9f5c360 100644 --- a/test/dictionary-data.write.js +++ b/test/dictionary-data.write.js @@ -21,8 +21,11 @@ import path from 'path'; import {parseJson} from '../dev/json.js'; import {createTranslatorTest} from './fixtures/translator-test.js'; import {createTestAnkiNoteData, getTemplateRenderResults} from './utilities/anki.js'; +import {setupStubs} from './utilities/database.js'; import {createFindKanjiOptions, createFindTermsOptions} from './utilities/translator.js'; +setupStubs(); + /** * @param {string} fileName * @param {unknown} content diff --git a/test/jsconfig.json b/test/jsconfig.json index 3e3eb57c2a..6640cb690d 100644 --- a/test/jsconfig.json +++ b/test/jsconfig.json @@ -22,11 +22,18 @@ }, "types": [ "chrome", + "dom-webcodecs", "firefox-webext-browser", "handlebars", "jszip", "parse5", "wanakana" + ], + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable", + "WebWorker" ] }, "include": [ diff --git a/test/utilities/database.js b/test/utilities/database.js new file mode 100644 index 0000000000..4fdf93ad0d --- /dev/null +++ b/test/utilities/database.js @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2023-2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +import {vi} from 'vitest'; + +/** + * + */ +export function setupStubs() { + vi.stubGlobal('self', { + constructor: { + name: 'Window', + }, + }); + + // eslint-disable-next-line jsdoc/require-jsdoc + function Worker() { + return { + addEventListener: () => {}, + }; + } + vi.stubGlobal('Worker', Worker); +} diff --git a/types/ext/api.d.ts b/types/ext/api.d.ts index 45996c48c2..7ff5b1d32f 100644 --- a/types/ext/api.d.ts +++ b/types/ext/api.d.ts @@ -418,3 +418,77 @@ type ApiMessage = { action: TName; params: ApiParams; }; + +// postMessage API (i.e., API endpoints called via postMessage, either through ServiceWorker on Chrome or a MessageChannel port on Firefox) + +type PmApiSurface = { + drawMedia: { + params: { + requests: DrawMediaRequest[]; + }; + return: void; + }; + connectToDatabaseWorker: { + params: void; + return: void; + }; + drawBufferToCanvases: { + params: { + buffer: ArrayBuffer; + width: number; + height: number; + canvasIndexes: number[]; + generation: number; + }; + return: void; + }; + drawDecodedImageToCanvases: { + params: { + decodedImage: VideoFrame | ImageBitmap; + canvasIndexes: number[]; + generation: number; + }; + return: void; + }; + registerOffscreenPort: { + params: void; + return: void; + }; + registerDatabasePort: { + params: void; + return: void; + }; +}; + +type DrawMediaRequest = { + path: string; + dictionary: string; + canvas: OffscreenCanvas; +}; + +type PmApiExtraArgs = [ports: readonly MessagePort[] | null]; + +export type PmApiNames = BaseApiNames; + +export type PmApiMap = BaseApiMap; + +export type PmApiMapInit = BaseApiMapInit; + +export type PmApiHandler = BaseApiHandler; + +export type PmApiHandlerNoExtraArgs = BaseApiHandler; + +export type PmApiParams = BaseApiParams; + +export type PmApiParam> = BaseApiParam; + +export type PmApiReturn = BaseApiReturn; + +export type PmApiParamsAny = BaseApiParamsAny; + +export type PmApiMessageAny = {[name in PmApiNames]: PmApiMessage}[PmApiNames]; + +type PmApiMessage = { + action: TName; + params: PmApiParams; +}; diff --git a/types/ext/dictionary-database-worker-handler.d.ts b/types/ext/dictionary-database-worker-handler.d.ts new file mode 100644 index 0000000000..67630c00a2 --- /dev/null +++ b/types/ext/dictionary-database-worker-handler.d.ts @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2023-2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +export type MessageToWorker = ( + ConnectToDatabaseWorker +); + +export type ConnectToDatabaseWorker = { + action: 'connectToDatabaseWorker'; + params: null; +}; diff --git a/types/ext/dictionary-database.d.ts b/types/ext/dictionary-database.d.ts index 84c6da625e..69078c114d 100644 --- a/types/ext/dictionary-database.d.ts +++ b/types/ext/dictionary-database.d.ts @@ -36,10 +36,12 @@ export type MediaDataArrayBufferContent = MediaDataBase; export type MediaDataStringContent = MediaDataBase; -type MediaType = ArrayBuffer | string; +type MediaType = ArrayBuffer | string | null; export type Media = {index: number} & MediaDataBase; +export type DrawMedia = {index: number} & MediaDataBase & {canvasWidth: number, canvasIndexes: number[], generation: number}; + export type DatabaseTermEntry = { expression: string; reading: string; @@ -239,6 +241,24 @@ export type MediaRequest = { dictionary: string; }; +export type DrawMediaRequest = { + path: string; + dictionary: string; + canvasIndex: number; + canvasWidth: number; + canvasHeight: number; + generation: number; +}; + +export type DrawMediaGroupedRequest = { + path: string; + dictionary: string; + canvasIndexes: number[]; + canvasWidth: number; + canvasHeight: number; + generation: number; +}; + export type FindMultiBulkData = { item: TItem; itemIndex: number; @@ -254,3 +274,57 @@ export type CreateResult = ( export type DictionarySet = { has(value: string): boolean; }; + +/** API for communicating with its own worker */ + +import type { + ApiMap as BaseApiMap, + ApiMapInit as BaseApiMapInit, + ApiHandler as BaseApiHandler, + ApiParams as BaseApiParams, + ApiReturn as BaseApiReturn, + ApiNames as BaseApiNames, + ApiParam as BaseApiParam, + ApiParamNames as BaseApiParamNames, + ApiParamsAny as BaseApiParamsAny, +} from './api-map'; + +type ApiSurface = { + drawMedia: { + params: { + requests: DrawMediaRequest[]; + }; + return: void; + }; + dummy: { + params: void; + return: void; + }; +}; + +type ApiExtraArgs = [port: MessagePort]; + +export type ApiNames = BaseApiNames; + +export type ApiMap = BaseApiMap; + +export type ApiMapInit = BaseApiMapInit; + +export type ApiHandler = BaseApiHandler; + +export type ApiHandlerNoExtraArgs = BaseApiHandler; + +export type ApiParams = BaseApiParams; + +export type ApiParam> = BaseApiParam; + +export type ApiReturn = BaseApiReturn; + +export type ApiParamsAny = BaseApiParamsAny; + +export type ApiMessageAny = {[name in ApiNames]: ApiMessage}[ApiNames]; + +type ApiMessage = { + action: TName; + params: ApiParams; +}; diff --git a/types/ext/display-content-manager.d.ts b/types/ext/display-content-manager.d.ts index a216fce9b7..0f9ee83435 100644 --- a/types/ext/display-content-manager.d.ts +++ b/types/ext/display-content-manager.d.ts @@ -15,26 +15,28 @@ * along with this program. If not, see . */ -import type * as DictionaryDatabase from './dictionary-database'; - /** A callback used when a media file has been loaded. */ export type OnLoadCallback = ( /** The URL of the media that was loaded. */ url: string, -) => void; +) => Promise; /** A callback used when a media file should be unloaded. */ export type OnUnloadCallback = ( /** Whether or not the media was fully loaded. */ fullyLoaded: boolean, -) => void; - -export type CachedMediaDataLoaded = { - data: DictionaryDatabase.MediaDataStringContent; - url: string; -}; +) => Promise; export type LoadMediaDataInfo = { onUnload: OnUnloadCallback; loaded: boolean; }; + +export type LoadMediaRequest = { + /** The path to the media file in the dictionary. */ + path: string; + /** The name of the dictionary. */ + dictionary: string; + /** The canvas to draw the image onto. */ + canvas: OffscreenCanvas; +}; diff --git a/types/ext/offscreen.d.ts b/types/ext/offscreen.d.ts index 2f180e215c..e9d6199265 100644 --- a/types/ext/offscreen.d.ts +++ b/types/ext/offscreen.d.ts @@ -95,6 +95,10 @@ type ApiSurface = { params: void; return: string | null; }; + createAndRegisterPortOffscreen: { + params: void; + return: void; + }; }; export type ApiMessage = ( @@ -136,3 +140,38 @@ export type ApiParams = BaseApiParams export type ApiReturn = BaseApiReturn; export type ApiMessageAny = {[name in ApiNames]: ApiMessage}[ApiNames]; + +// MessageChannel API + +type McApiSurface = { + connectToDatabaseWorker: { + params: void; + return: void; + }; + dummy: { + params: void; + return: void; + }; +}; + +type McApiExtraArgs = [ports: readonly MessagePort[]]; + +export type McApiMessage = ( + McApiParams extends void ? + {action: TName, params?: never} : + {action: TName, params: McApiParams} +); + +export type McApiNames = BaseApiNames; + +export type McApiMap = BaseApiMap; + +export type McApiMapInit = BaseApiMapInit; + +export type McApiHandler = BaseApiHandler; + +export type McApiParams = BaseApiParams; + +export type McApiReturn = BaseApiReturn; + +export type McApiMessageAny = {[name in McApiNames]: McApiMessage}[McApiNames]; diff --git a/types/ext/shared-worker.d.ts b/types/ext/shared-worker.d.ts new file mode 100644 index 0000000000..936f964114 --- /dev/null +++ b/types/ext/shared-worker.d.ts @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2023-2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import type { + ApiMap as BaseApiMap, + ApiMapInit as BaseApiMapInit, + ApiHandler as BaseApiHandler, + ApiParams as BaseApiParams, + ApiReturn as BaseApiReturn, + ApiNames as BaseApiNames, + ApiParam as BaseApiParam, + ApiParamNames as BaseApiParamNames, + ApiParamsAny as BaseApiParamsAny, +} from './api-map'; + +type ApiSurface = { + registerBackendPort: { + params: void; + return: void; + }; + connectToBackend1: { + params: void; + return: void; + }; +}; + +type ApiExtraArgs = [interlocutorPort: MessagePort, ports: readonly MessagePort[]]; + +export type ApiNames = BaseApiNames; + +export type ApiMap = BaseApiMap; + +export type ApiMapInit = BaseApiMapInit; + +export type ApiHandler = BaseApiHandler; + +export type ApiHandlerNoExtraArgs = BaseApiHandler; + +export type ApiParams = BaseApiParams; + +export type ApiParam> = BaseApiParam; + +export type ApiReturn = BaseApiReturn; + +export type ApiParamsAny = BaseApiParamsAny; + +export type ApiMessageAny = {[name in ApiNames]: ApiMessage}[ApiNames]; + +type ApiMessage = { + action: TName; + params: ApiParams; +};