From 0f0152091d9a6b64d9cfc5174f8ddd844018485a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Sun, 1 Dec 2024 22:13:07 +0100 Subject: [PATCH] Blueprints: Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build (#2027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/WordPress/wordpress-playground/pull/1987 created a dependency between the `@wp-playground/blueprints` package and the `@wp-playground/wordpress-builds` package breaking `@wp-playground/cli` – see https://github.com/WordPress/wordpress-playground/issues/2026 This PR updates the setSiteLanguage step to pull the latest/best WordPress version details from api.wordpress.org/core/version-check/1.7?channel=beta instead of implicitly assuming the latest version is the same as that of the latest minified web build. It reuses the Playground CLI resolveWordPressRelease function, bringing us closer to having zero CLI-specific logic. ## Follow-up work Invent an ESLint rule to prevent further dependencies on `@wp-playground/wordpress-builds` ## Testing instructions * CI tests * Run Playground CLI via `bun packages/playground/cli/src/cli.ts server` and confirm WordPress is still being downloaded without errors. cc @swissspidy @bgrgicak --- .github/workflows/ci.yml | 4 + .../src/lib/steps/set-site-language.spec.ts | 4 +- .../src/lib/steps/set-site-language.ts | 171 +++++++++++------- .../playground/blueprints/tsconfig.spec.json | 3 +- packages/playground/cli/src/cli.ts | 5 +- packages/playground/cli/src/download.ts | 60 +----- .../src}/create-memoized-fetch.ts | 0 packages/playground/common/src/index.ts | 2 + .../src/test}/create-memoized-fetch.spec.ts | 2 +- .../remote/src/lib/worker-thread.ts | 2 +- packages/playground/remote/tsconfig.lib.json | 4 +- packages/playground/remote/tsconfig.spec.json | 3 +- .../website/playwright/e2e/blueprints.spec.ts | 2 +- packages/playground/wordpress/src/index.ts | 94 +++++++++- 14 files changed, 225 insertions(+), 131 deletions(-) rename packages/playground/{remote/src/lib => common/src}/create-memoized-fetch.ts (100%) rename packages/playground/{remote/src/lib => common/src/test}/create-memoized-fetch.spec.ts (93%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab3a483232..fa3ce050d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,10 @@ jobs: name: playwright-snapshots-${{ matrix.part }} path: packages/playground/website/playwright/e2e/deployment.spec.ts-snapshots/ if-no-files-found: ignore + - name: Delete playwright-dist artifact + uses: geekyeggo/delete-artifact@v2 + with: + name: playwright-dist build: runs-on: ubuntu-latest diff --git a/packages/playground/blueprints/src/lib/steps/set-site-language.spec.ts b/packages/playground/blueprints/src/lib/steps/set-site-language.spec.ts index e022ec6e3e..ac8c03447b 100644 --- a/packages/playground/blueprints/src/lib/steps/set-site-language.spec.ts +++ b/packages/playground/blueprints/src/lib/steps/set-site-language.spec.ts @@ -20,7 +20,7 @@ describe('getTranslationUrl()', () => { }, { versionString: '6.6-RC1', - latestBetaVersion: '6.6-RC', + latestBetaVersion: '6.6-RC1', latestMinifiedVersion: '6.5.2', expectedUrl: `https://downloads.wordpress.org/translation/core/6.6-RC/en_US.zip`, description: @@ -66,7 +66,7 @@ describe('getTranslationUrl()', () => { latestBetaVersion, latestMinifiedVersion ) - ).toBe(expectedUrl); + ).resolves.toBe(expectedUrl); }); } ); diff --git a/packages/playground/blueprints/src/lib/steps/set-site-language.ts b/packages/playground/blueprints/src/lib/steps/set-site-language.ts index f69ebd8d1f..4bac53be96 100644 --- a/packages/playground/blueprints/src/lib/steps/set-site-language.ts +++ b/packages/playground/blueprints/src/lib/steps/set-site-language.ts @@ -1,10 +1,8 @@ import { StepHandler } from '.'; import { unzipFile } from '@wp-playground/common'; import { logger } from '@php-wasm/logger'; -import { - LatestMinifiedWordPressVersion, - MinifiedWordPressVersions, -} from '@wp-playground/wordpress-builds'; +import { resolveWordPressRelease } from '@wp-playground/wordpress'; +import { Semaphore } from '@php-wasm/util'; /** * @inheritDoc setSiteLanguage @@ -25,48 +23,79 @@ export interface SetSiteLanguageStep { } /** - * Returns the URL to download a WordPress translation package. + * Infers the translation package URL for a given WordPress version. * - * If the WordPress version doesn't have a translation package, - * the latest "RC" version will be used instead. + * If it cannot be inferred, the latest translation package will be used instead. */ -export const getWordPressTranslationUrl = ( +export const getWordPressTranslationUrl = async ( wpVersion: string, language: string, - latestBetaVersion: string = MinifiedWordPressVersions['beta'], - latestMinifiedVersion: string = LatestMinifiedWordPressVersion + latestBetaWordPressVersion?: string, + latestStableWordPressVersion?: string ) => { /** - * The translation API provides translations for all WordPress releases - * including patch releases. + * Infer a WordPress version we can feed into the translations API based + * on the requested fully-qualified WordPress version. * - * RC and beta versions don't have individual translation packages. - * They all share the same "RC" translation package. + * The translation API provides translations for: * - * Nightly versions don't have a "nightly" translation package. - * So, the best we can do is download the RC translation package, - * because it contains the latest available translations. + * - all major.minor WordPress releases + * - all major.minor.patch WordPress releases + * - Latest beta/RC version – under a label like "6.6-RC". It's always "-RC". + * There's no "-BETA1", "-RC1", "-RC2", etc. * - * The WordPress.org translation API uses "RC" instead of - * "RC1", "RC2", "BETA1", "BETA2", etc. + * The API does not provide translations for "nightly", "latest", or + * old beta/RC versions. * * For example translations for WordPress 6.6-BETA1 or 6.6-RC1 are found under * https://downloads.wordpress.org/translation/core/6.6-RC/en_GB.zip */ - if (wpVersion.match(/^(\d.\d(.\d)?)-(alpha|beta|nightly|rc).*$/i)) { - wpVersion = latestBetaVersion + let resolvedVersion = null; + if (wpVersion.match(/^(\d+\.\d+)(?:\.\d+)?$/)) { + // Use the version directly if it's a major.minor or major.minor.patch. + resolvedVersion = wpVersion; + } else if (wpVersion.match(/^(\d.\d(.\d)?)-(beta|rc|alpha|nightly).*$/i)) { + // Translate "6.4-alpha", "6.5-beta", "6.6-nightly", "6.6-RC" etc. + // to "6.6-RC" + if (latestBetaWordPressVersion) { + resolvedVersion = latestBetaWordPressVersion; + } else { + let resolved = await resolveWordPressRelease('beta'); + // Beta versions are only available during the beta period – + // let's use the latest stable release as a fallback. + if (resolved.source !== 'api') { + resolved = await resolveWordPressRelease('latest'); + } + resolvedVersion = resolved!.version; + } + resolvedVersion = resolvedVersion // Remove the patch version, e.g. 6.6.1-RC1 -> 6.6-RC1 .replace(/^(\d.\d)(.\d+)/i, '$1') // Replace "rc" and "beta" with "RC", e.g. 6.6-nightly -> 6.6-RC .replace(/(rc|beta).*$/i, 'RC'); - } else if (!wpVersion.match(/^(\d+\.\d+)(?:\.\d+)?$/)) { + } else { /** - * If the WordPress version string isn't a major.minor or major.minor.patch, - * the latest available WordPress build version will be used instead. + * Use the latest stable version otherwise. + * + * The requested version is neither stable, nor beta/RC, nor alpha/nightly. + * It must be a custom version string. We could actually fail at this point, + * but it seems more useful to* download translations from the last official + * WordPress version. If that assumption is wrong, let's reconsider this whenever + * someone reports a related issue. */ - wpVersion = latestMinifiedVersion; + if (latestStableWordPressVersion) { + resolvedVersion = latestStableWordPressVersion; + } else { + const resolved = await resolveWordPressRelease('latest'); + resolvedVersion = resolved!.version; + } } - return `https://downloads.wordpress.org/translation/core/${wpVersion}/${language}.zip`; + if (!resolvedVersion) { + throw new Error( + `WordPress version ${wpVersion} is not supported by the setSiteLanguage step` + ); + } + return `https://downloads.wordpress.org/translation/core/${resolvedVersion}/${language}.zip`; }; /** @@ -94,7 +123,7 @@ export const setSiteLanguage: StepHandler = async ( const translations = [ { - url: getWordPressTranslationUrl(wpVersion, language), + url: await getWordPressTranslationUrl(wpVersion, language), type: 'core', }, ]; @@ -165,43 +194,61 @@ export const setSiteLanguage: StepHandler = async ( await playground.mkdir(`${docroot}/wp-content/languages/themes`); } - for (const { url, type } of translations) { - try { - const response = await fetch(url); - if (!response.ok) { - throw new Error( - `Failed to download translations for ${type}: ${response.statusText}` - ); - } + // Fetch translations in parallel + const fetchQueue = new Semaphore({ concurrency: 5 }); + const translationsQueue = translations.map(({ url, type }) => + fetchQueue.run(async () => { + try { + const response = await fetch(url); + if (!response.ok) { + throw new Error( + `Failed to download translations for ${type}: ${response.statusText}` + ); + } - let destination = `${docroot}/wp-content/languages`; - if (type === 'plugin') { - destination += '/plugins'; - } else if (type === 'theme') { - destination += '/themes'; - } + let destination = `${docroot}/wp-content/languages`; + if (type === 'plugin') { + destination += '/plugins'; + } else if (type === 'theme') { + destination += '/themes'; + } - await unzipFile( - playground, - new File([await response.blob()], `${language}-${type}.zip`), - destination - ); - } catch (error) { - /** - * If a core translation wasn't found we should throw an error because it - * means the language is not supported or the language code isn't correct. - */ - if (type === 'core') { - throw new Error( - `Failed to download translations for WordPress. Please check if the language code ${language} is correct. You can find all available languages and translations on https://translate.wordpress.org/.` + await unzipFile( + playground, + new File( + [await response.blob()], + `${language}-${type}.zip` + ), + destination + ); + } catch (error) { + /** + * Throw an error when a core translation isn't found. + * + * The language slug used in the Blueprint is not recognized by the + * WordPress.org API and will always return a 404. This is likely + * unintentional – perhaps a typo or the API consumer guessed the + * slug wrong. + * + * The least we can do is communicate the problem. + */ + if (type === 'core') { + throw new Error( + `Failed to download translations for WordPress. Please check if the language code ${language} is correct. You can find all available languages and translations on https://translate.wordpress.org/.` + ); + } + /** + * WordPress core has translations for the requested language, + * but one of the installed plugins or themes doesn't. + * + * This is fine. Not all plugins and themes have translations for + * every language. Let's just log a warning and move on. + */ + logger.warn( + `Error downloading translations for ${type}: ${error}` ); } - /** - * Some languages don't have translations for themes and plugins and will - * return a 404 and a CORS error. In this case, we can just skip the - * download because Playground can still work without them. - */ - logger.warn(`Error downloading translations for ${type}: ${error}`); - } - } + }) + ); + await Promise.all(translationsQueue); }; diff --git a/packages/playground/blueprints/tsconfig.spec.json b/packages/playground/blueprints/tsconfig.spec.json index eb23daacbc..06268eb779 100644 --- a/packages/playground/blueprints/tsconfig.spec.json +++ b/packages/playground/blueprints/tsconfig.spec.json @@ -14,6 +14,7 @@ "src/**/*.spec.js", "src/**/*.test.jsx", "src/**/*.spec.jsx", - "src/**/*.d.ts" + "src/**/*.d.ts", + "../common/src/**/*.spec.ts" ] } diff --git a/packages/playground/cli/src/cli.ts b/packages/playground/cli/src/cli.ts index 9b2fd3b23e..bdcbbb5ace 100644 --- a/packages/playground/cli/src/cli.ts +++ b/packages/playground/cli/src/cli.ts @@ -27,9 +27,8 @@ import { fetchSqliteIntegration, fetchWordPress, readAsFile, - resolveWPRelease, } from './download'; - +import { resolveWordPressRelease } from '@wp-playground/wordpress'; export interface Mount { hostPath: string; vfsPath: string; @@ -281,7 +280,7 @@ async function run() { } }) as any); - wpDetails = await resolveWPRelease(args.wp); + wpDetails = await resolveWordPressRelease(args.wp); } const preinstalledWpContentPath = diff --git a/packages/playground/cli/src/download.ts b/packages/playground/cli/src/download.ts index 54ab1d305e..7a73e63e65 100644 --- a/packages/playground/cli/src/download.ts +++ b/packages/playground/cli/src/download.ts @@ -1,5 +1,5 @@ import { EmscriptenDownloadMonitor } from '@php-wasm/progress'; -import { createHash } from 'crypto'; +import { resolveWordPressRelease } from '@wp-playground/wordpress'; import fs from 'fs-extra'; import os from 'os'; import path, { basename } from 'path'; @@ -16,9 +16,9 @@ export async function fetchWordPress( wpVersion = 'latest', monitor: EmscriptenDownloadMonitor ) { - const wpDetails = await resolveWPRelease(wpVersion); + const wpDetails = await resolveWordPressRelease(wpVersion); const wpZip = await cachedDownload( - wpDetails.url, + wpDetails.releaseUrl, `${wpDetails.version}.zip`, monitor ); @@ -88,57 +88,3 @@ async function downloadTo( export function readAsFile(path: string, fileName?: string): File { return new File([fs.readFileSync(path)], fileName ?? basename(path)); } - -export async function resolveWPRelease(version = 'latest') { - // Support custom URLs - if (version.startsWith('https://') || version.startsWith('http://')) { - const shasum = createHash('sha1'); - shasum.update(version); - const sha1 = shasum.digest('hex'); - return { - url: version, - version: 'custom-' + sha1.substring(0, 8), - }; - } - - if (version === 'trunk' || version === 'nightly') { - return { - url: 'https://wordpress.org/nightly-builds/wordpress-latest.zip', - version: 'nightly-' + new Date().toISOString().split('T')[0], - }; - } - - let latestVersions = await fetch( - 'https://api.wordpress.org/core/version-check/1.7/?channel=beta' - ).then((res) => res.json()); - - latestVersions = latestVersions.offers.filter( - (v: any) => v.response === 'autoupdate' - ); - - for (const apiVersion of latestVersions) { - if (version === 'beta' && apiVersion.version.includes('beta')) { - return { - url: apiVersion.download, - version: apiVersion.version, - }; - } else if (version === 'latest') { - return { - url: apiVersion.download, - version: apiVersion.version, - }; - } else if ( - apiVersion.version.substring(0, version.length) === version - ) { - return { - url: apiVersion.download, - version: apiVersion.version, - }; - } - } - - return { - url: `https://wordpress.org/wordpress-${version}.zip`, - version: version, - }; -} diff --git a/packages/playground/remote/src/lib/create-memoized-fetch.ts b/packages/playground/common/src/create-memoized-fetch.ts similarity index 100% rename from packages/playground/remote/src/lib/create-memoized-fetch.ts rename to packages/playground/common/src/create-memoized-fetch.ts diff --git a/packages/playground/common/src/index.ts b/packages/playground/common/src/index.ts index ad95b489e9..3c9b6c4d95 100644 --- a/packages/playground/common/src/index.ts +++ b/packages/playground/common/src/index.ts @@ -12,6 +12,8 @@ import { UniversalPHP } from '@php-wasm/universal'; import { phpVars } from '@php-wasm/util'; +export { createMemoizedFetch } from './create-memoized-fetch'; + export const RecommendedPHPVersion = '8.0'; /** diff --git a/packages/playground/remote/src/lib/create-memoized-fetch.spec.ts b/packages/playground/common/src/test/create-memoized-fetch.spec.ts similarity index 93% rename from packages/playground/remote/src/lib/create-memoized-fetch.spec.ts rename to packages/playground/common/src/test/create-memoized-fetch.spec.ts index b6dd6919ad..8a0a6bc1a8 100644 --- a/packages/playground/remote/src/lib/create-memoized-fetch.spec.ts +++ b/packages/playground/common/src/test/create-memoized-fetch.spec.ts @@ -1,4 +1,4 @@ -import { createMemoizedFetch } from './create-memoized-fetch'; +import { createMemoizedFetch } from '../create-memoized-fetch'; describe('createMemoizedFetch', () => { it('should return a function', () => { diff --git a/packages/playground/remote/src/lib/worker-thread.ts b/packages/playground/remote/src/lib/worker-thread.ts index 04d14d8b47..8bdddd346e 100644 --- a/packages/playground/remote/src/lib/worker-thread.ts +++ b/packages/playground/remote/src/lib/worker-thread.ts @@ -25,7 +25,7 @@ import { hasCachedStaticFilesRemovedFromMinifiedBuild, } from './worker-utils'; import { EmscriptenDownloadMonitor } from '@php-wasm/progress'; -import { createMemoizedFetch } from './create-memoized-fetch'; +import { createMemoizedFetch } from '@wp-playground/common'; import { FilesystemOperation, journalFSEvents, diff --git a/packages/playground/remote/tsconfig.lib.json b/packages/playground/remote/tsconfig.lib.json index bc7e4da799..15dd1d55b6 100644 --- a/packages/playground/remote/tsconfig.lib.json +++ b/packages/playground/remote/tsconfig.lib.json @@ -13,7 +13,9 @@ "src/**/*.ts", "service-worker.ts", "../../php-wasm/web/src/lib/tls/1_2/connection.ts", - "../../php-wasm/web/src/lib/tls/certificates.ts" + "../../php-wasm/web/src/lib/tls/certificates.ts", + "../common/src/create-memoized-fetch.spec.ts", + "../common/src/create-memoized-fetch.ts" ], "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] } diff --git a/packages/playground/remote/tsconfig.spec.json b/packages/playground/remote/tsconfig.spec.json index fead792f73..b24202026c 100644 --- a/packages/playground/remote/tsconfig.spec.json +++ b/packages/playground/remote/tsconfig.spec.json @@ -21,6 +21,7 @@ "src/**/*.spec.js", "src/**/*.test.jsx", "src/**/*.spec.jsx", - "src/**/*.d.ts" + "src/**/*.d.ts", + "../common/src/create-memoized-fetch.spec.ts" ] } diff --git a/packages/playground/website/playwright/e2e/blueprints.spec.ts b/packages/playground/website/playwright/e2e/blueprints.spec.ts index ba650660a5..18a3a0a902 100644 --- a/packages/playground/website/playwright/e2e/blueprints.spec.ts +++ b/packages/playground/website/playwright/e2e/blueprints.spec.ts @@ -442,7 +442,7 @@ test('should correctly redirect to a multisite wp-admin url', async ({ const blueprint: Blueprint = { landingPage: '/wp-admin/', preferredVersions: { - wp: version, + wp: 'nightly', }, steps: [{ step: 'setSiteLanguage', language: 'es_ES' }], }; diff --git a/packages/playground/wordpress/src/index.ts b/packages/playground/wordpress/src/index.ts index a26d7aa90d..7eb8c9f0e3 100644 --- a/packages/playground/wordpress/src/index.ts +++ b/packages/playground/wordpress/src/index.ts @@ -1,6 +1,6 @@ import { PHP, UniversalPHP } from '@php-wasm/universal'; import { joinPaths, phpVar } from '@php-wasm/util'; -import { unzipFile } from '@wp-playground/common'; +import { unzipFile, createMemoizedFetch } from '@wp-playground/common'; export { bootWordPress, getFileNotFoundActionForWordPress } from './boot'; export { getLoadedWordPressVersion } from './version-detect'; @@ -545,3 +545,95 @@ function isCleanDirContainingSiteMetadata(path: string, php: PHP) { return false; } + +const memoizedFetch = createMemoizedFetch(fetch); + +/** + * Resolves a specific WordPress release URL and version string based on + * a version query string such as "latest", "beta", or "6.6". + * + * Examples: + * ```js + * const { releaseUrl, version } = await resolveWordPressRelease('latest') + * // becomes https://wordpress.org/wordpress-6.6.2.zip and '6.6.2' + * + * const { releaseUrl, version } = await resolveWordPressRelease('beta') + * // becomes https://wordpress.org/wordpress-6.6.2-RC1.zip and '6.6.2-RC1' + * + * const { releaseUrl, version } = await resolveWordPressRelease('6.6') + * // becomes https://wordpress.org/wordpress-6.6.2.zip and '6.6.2' + * ``` + * + * @param versionQuery - The WordPress version query string to resolve. + * @returns The resolved WordPress release URL and version string. + */ +export async function resolveWordPressRelease(versionQuery = 'latest') { + if ( + versionQuery.startsWith('https://') || + versionQuery.startsWith('http://') + ) { + const shasum = await crypto.subtle.digest( + 'SHA-1', + new TextEncoder().encode(versionQuery) + ); + const sha1 = Array.from(new Uint8Array(shasum)) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); + return { + releaseUrl: versionQuery, + version: 'custom-' + sha1.substring(0, 8), + source: 'inferred', + }; + } else if (versionQuery === 'trunk' || versionQuery === 'nightly') { + return { + releaseUrl: + 'https://wordpress.org/nightly-builds/wordpress-latest.zip', + version: 'nightly-' + new Date().toISOString().split('T')[0], + source: 'inferred', + }; + } + + const response = await memoizedFetch( + 'https://api.wordpress.org/core/version-check/1.7/?channel=beta' + ); + let latestVersions = await response.json(); + + latestVersions = latestVersions.offers.filter( + (v: any) => v.response === 'autoupdate' + ); + + for (const apiVersion of latestVersions) { + if (versionQuery === 'beta' && apiVersion.version.includes('beta')) { + return { + releaseUrl: apiVersion.download, + version: apiVersion.version, + source: 'api', + }; + } else if ( + versionQuery === 'latest' && + !apiVersion.version.includes('beta') + ) { + // The first non-beta item in the list is the latest version. + return { + releaseUrl: apiVersion.download, + version: apiVersion.version, + source: 'api', + }; + } else if ( + apiVersion.version.substring(0, versionQuery.length) === + versionQuery + ) { + return { + releaseUrl: apiVersion.download, + version: apiVersion.version, + source: 'api', + }; + } + } + + return { + releaseUrl: `https://wordpress.org/wordpress-${versionQuery}.zip`, + version: versionQuery, + source: 'inferred', + }; +}