chore(l10n): extract strings, add docs, load locales on demand

README.md

@@ -24,6 +24,10 @@ MDN's next fr(ont)e(n)d.
 - `npm run preview`
   - runs the preview server: using the production bundles with the rari server: useful for testing our prod rspack config
 
+## L10n
+
+See [the l10n README](./l10n/README.md).
+
 ## Development principles
 
 ### Inline JS

build/plugins/extract-l10n.js

@@ -0,0 +1,16 @@
+import { extract } from "../../l10n/parser/extractor.js";
+
+/**
+ * @import { Compiler } from "@rspack/core"
+ */
+
+export class ExtractL10nPlugin {
+  /**
+   * @param {Compiler} compiler
+   */
+  apply(compiler) {
+    compiler.hooks.beforeCompile.tapPromise("ExtractL10nPlugin", async () => {
+      await extract();
+    });
+  }
+}

entry.ssr.js

@@ -45,9 +45,6 @@ for (const [name, def] of customElements.__definitions) {
  */
 export async function render(path, partialContext, compilationStats) {
   const locale = path.split("/")[1] || "en-US";
-  if (locale === "qa") {
-    path = path.replace("/qa/", "/en-US/");
-  }
 
   const context = {
     path,

l10n/README.md

@@ -0,0 +1,71 @@
+# L10n
+
+We use [Fluent](https://projectfluent.org/) for l10n.
+
+## For developers
+
+In order to make adding l10n strings easy, we have a couple of convenience methods exposed via `context.l10n` (in a server component) and `this.l10n` (in a custom element, with the `L10nMixin` applied).
+
+These can be used in a few ways:
+
+```js
+// As a simple inline string, with no ID: it'll be
+// automatically generated
+context.l10n`Hello`;
+
+// As an inline string, with a manually specified ID:
+// necessary if you need to disambiguate between two
+// instances of the same string in en-US, which aren't
+// the same in other locales
+context.l10n("hello")`Hello`;
+
+// For more complex scenarios, involving arguments or
+// HTML within the string:
+context.l10n.raw({
+  id: "hello-person",
+  args: {
+    name: "world",
+  },
+  elements: {
+    link: { tag: "a", href: "https://example.com/" },
+  },
+});
+```
+
+The last of those examples requires manually adding a string to `./locales/en-US.ftl`, like this:
+
+```ftl
+hello-person = Hello <a data-l10n-name="link">{ $name }</a>!
+```
+
+And it'll render into HTML like this:
+
+```html
+Hello <a href="https://example.com/">world</a>!
+```
+
+The other examples are automatically extracted, and combined with the manually specified strings, into `./template.ftl`.
+
+### Pseudolocalization
+
+We have a few pseudo locales for testing if strings have been added, or if components will adapt to localized strings:
+
+- `qaa`: "accented" locale: adds accents to all characters, duplicates some vowels to create longer strings, wraps string in square brackets to help detect truncation
+- `qai`: "id" locale: replaces strings with their identifiers, wrapped in square brackets
+
+The `qai` locale works all the time, the `qaa` locale must be manually generated with `node ./parser/transform.js`
+
+## For localizers
+
+The l10n experience isn't fantastic at the moment, and we have improvements to make, but it should be functional.
+
+Strings to be localized can be sourced from `./template.ftl`: this will include both manually added strings, as well as strings scraped from the code. Localized strings should be placed in `./locales/{locale}.ftl`.
+
+Adding Fluent comments to explain what context strings appear in is an open task, but for now they can be found in code:
+
+- if the string has an autogenerated ID (ending in a hash), search for the English string in code: it'll appear as `` {this|context}.l10n`{the string}` ``
+- if the string has manual ID, search for the ID in code: it'll appear as `{this|context}.l10n({the id})` or `{this|context}.l10n({ id: {the id} })`
+
+If one English string is used in multiple places and requires multiple different strings in a locale, file an issue to manually add IDs for each string.
+
+Please also file an issue for any other problems you encounter with localizing - either with specific strings which need fixing, or general issues with the l10n process.

l10n/context.js

@@ -1,10 +1,11 @@
-import getFluentContext from "./fluent.js";
+import getFluentContext, { loadFluentFile } from "./fluent.js";
 
 /**
  * @param {string} locale
  * @returns {Promise<import("@fred").L10nContext>}
  */
 export async function addFluent(locale) {
+  await loadFluentFile(locale);
   return {
     locale: locale,
     l10n: getFluentContext(locale),

l10n/djb2a.js

@@ -0,0 +1,33 @@
+/* eslint-disable unicorn/prefer-code-point */
+/**
+ * djb2a based on https://github.com/sindresorhus/djb2a
+ *
+ * @license MIT
+ *
+ * Copyright (c) Sindre Sorhus <[email protected]> (https://sindresorhus.com)
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+// Explanation: https://stackoverflow.com/a/31621312/64949
+
+const MAGIC_CONSTANT = 5381;
+
+/**
+ * @param {string} string
+ */
+export default function djb2a(string) {
+  let hash = MAGIC_CONSTANT;
+
+  for (let index = 0; index < string.length; index++) {
+    // Equivalent to: `hash * 33 ^ string.charCodeAt(i)`
+    hash = ((hash << 5) + hash) ^ string.charCodeAt(index);
+  }
+
+  // Convert it to an unsigned 32-bit integer.
+  return hash >>> 0;
+}

l10n/fluent.js

@@ -2,33 +2,16 @@ import { FluentBundle, FluentResource } from "@fluent/bundle";
 import insane from "insane";
 import { unsafeHTML } from "lit/directives/unsafe-html.js";
 
-import de_ftl from "../l10n/de.ftl";
-import enUS_ftl from "../l10n/en-US.ftl";
-import es_ftl from "../l10n/es.ftl";
-import fr_ftl from "../l10n/fr.ftl";
-import ja_ftl from "../l10n/ja.ftl";
-import ko_ftl from "../l10n/ko.ftl";
-import ptBR_ftl from "../l10n/pt-BR.ftl";
-import ru_ftl from "../l10n/ru.ftl";
-import zhCN_ftl from "../l10n/zh-CN.ftl";
-import zhTW_ftl from "../l10n/zh-TW.ftl";
+import enUS_ftl from "./locales/en-US.ftl";
+import { generateIdFromString } from "./utils.js";
 
 /**
  * @import { AllowedTags } from "insane";
  */
 
 /** @type {Record<string, string>} */
-const ftlMap = {
+let ftlMap = {
   "en-US": enUS_ftl,
-  de: de_ftl,
-  es: es_ftl,
-  fr: fr_ftl,
-  ja: ja_ftl,
-  ko: ko_ftl,
-  "pt-BR": ptBR_ftl,
-  ru: ru_ftl,
-  "zh-CN": zhCN_ftl,
-  "zh-TW": zhTW_ftl,
 };
 
 const ALLOWED_TAGS = ["i", "strong", "br", "em"];
@@ -149,7 +132,7 @@ export class Fluent {
     const parentMessage = bundle ? bundle.getMessage(id) : undefined;
     let message;
 
-    if (this.locale === "qa") {
+    if (this.locale === "qai") {
       return `[${id}${attr ? `.${attr}` : ""}]`;
     }
 
@@ -209,7 +192,23 @@ function getLocale(locale) {
 }
 
 /**
- * @param {string} [locale]
+ * @param {string} locale
+ */
+export async function loadFluentFile(locale) {
+  if (locale !== "qai" && !ftlMap[locale]) {
+    try {
+      const { default: localeStrings } = await import(
+        `./locales/${locale}.ftl`
+      );
+      ftlMap[locale] = localeStrings;
+    } catch (error) {
+      console.error(error);
+    }
+  }
+}
+
+/**
+ * @param {string} locale
  */
 export default function getFluentContext(locale) {
   /**
@@ -256,10 +255,14 @@ export default function getFluentContext(locale) {
     }
     // called directly as a template tag:
     // l10n`Foobar`
-    // TODO: create consistent logic for id generation at runtime and scrapetime
     const strings = idOrStrings;
-    const templateString = strings[0];
-    return templateString || "";
+    const templateString = strings[0] || "";
+    const id = generateIdFromString(templateString);
+    const localizedString = getLocale(locale)?.get(id);
+
+    return typeof localizedString === "string"
+      ? localizedString
+      : templateString;
   }
 
   /**

l10n/locales/.gitignore

@@ -0,0 +1 @@
+qa*.ftl

l10n/en-US.ftl → l10n/locales/en-US.ftl

@@ -1,3 +1,6 @@
+# WARNING: don't use this file as a source for strings requiring l10n, use ../template.ftl instead:
+# this file only contains manually added strings, not ones inlined in code. See ../README.md for more details.
+
 # TODO Use comments, see: https://firefox-source-docs.mozilla.org/l10n/fluent/review.html#comments
 # TODO Consider using terms, see: https://firefox-source-docs.mozilla.org/l10n/fluent/review.html#terms and https://projectfluent.org/fluent/guide/references.html#message-references
 

l10n/mixin.js

@@ -1,11 +1,14 @@
 import { getSymmetricContext } from "../symmetric-context/both.js";
 
-import getFluentContext from "./fluent.js";
+import getFluentContext, { loadFluentFile } from "./fluent.js";
 
 /**
  * @import { LitElement } from "lit";
  */
 
+const locale = getSymmetricContext()?.locale;
+if (locale) await loadFluentFile(locale);
+
 /**
  * @template {new (...args: any[]) => LitElement} TBase
  * @param {TBase} Base
@@ -17,7 +20,13 @@ export const L10nMixin = (Base) =>
      */
     constructor(...args) {
       super(...args);
-      const context = getSymmetricContext();
+      let context = getSymmetricContext();
+      if (!context) {
+        console.error("SymmetricContext is undefined, reverting to defaults");
+        context = {
+          locale: "en-US",
+        };
+      }
       this.locale = context.locale;
       this.l10n = getFluentContext(this.locale);
     }

l10n/parser/extract.js

@@ -0,0 +1,3 @@
+import { extract } from "./extractor.js";
+
+await extract();