h2m/m2h: l10n support for cards (mdn#4251)

* Adding l10n support for converter via node-gettext * Typo fr wording for warning cards * feat: add ko markdown card label * Fixes per review * Use shared constant for default locale * add l10n support for m2h * Fix merge typo & applied prettier * Fixing per @fiji-flo review * AFixing CI feedback - applied prettier * Fallback to en-US when no localization is provided * Fix callout label * prevent unexpected keyword to break / fix per Will's review Co-authored-by: hochan222 <[email protected]>
alaa-yahia · Sep 2, 2021 · 478f0e2 · 478f0e2
1 parent 5420d6c
commit 478f0e2
Show file tree

Hide file tree

Showing 13 changed files with 257 additions and 86 deletions.
diff --git a/kumascript/index.js b/kumascript/index.js
@@ -69,7 +69,9 @@ const renderFromURL = async (
   }
 
   const { rawBody, fileInfo, isMarkdown } = document;
-  const rawHTML = isMarkdown ? await m2h(rawBody) : rawBody;
+  const rawHTML = isMarkdown
+    ? await m2h(rawBody, { locale: metadata.locale })
+    : rawBody;
   const [renderedHtml, errors] = await renderMacros(
     rawHTML,
     {

diff --git a/kumascript/src/info.js b/kumascript/src/info.js
@@ -219,7 +219,7 @@ const info = {
         let $ = null;
         let summary = "";
         try {
-          $ = cheerio.load(isMarkdown ? m2hSync(rawBody) : rawBody);
+          $ = cheerio.load(isMarkdown ? m2hSync(rawBody, { locale }) : rawBody);
           $("span.seoSummary, .summary").each((i, element) => {
             if (!summary) {
               const html = $(element)

diff --git a/markdown/cli.ts b/markdown/cli.ts
@@ -104,6 +104,14 @@ function saveProblemsReport(problems: Map<any, any>) {
   }
 }
 
+function buildLocaleMap(locale) {
+  let localesMap = new Map();
+  if (locale !== "all") {
+    localesMap = new Map([[locale.toLowerCase(), locale]]);
+  }
+  return localesMap;
+}
+
 program
   .bin("yarn md")
   .name("md")
@@ -130,13 +138,9 @@ program
       console.log(
         `Starting HTML to Markdown conversion in ${options.mode} mode`
       );
-      let localesMap = new Map();
-      if (options.locale !== "all") {
-        localesMap = new Map([[options.locale.toLowerCase(), options.locale]]);
-      }
       const documents = Document.findAll({
         folderSearch: args.folder,
-        locales: localesMap,
+        locales: buildLocaleMap(options.locale),
       });
 
       const progressBar = new cliProgress.SingleBar(
@@ -167,6 +171,7 @@ program
           const { body: h, attributes: metadata } = fm(doc.rawContent);
           const [markdown, { invalid, unhandled }] = await h2m(h, {
             printAST: options.printAst,
+            locale: doc.metadata.locale,
           });
 
           if (invalid.length > 0 || unhandled.length > 0) {
@@ -206,16 +211,23 @@ program
   )
 
   .command("m2h", "Convert Markdown to HTML")
+  .option("--locale", "Targets a specific locale", {
+    default: "all",
+    validator: (Array.from(VALID_LOCALES.values()) as string[]).concat("all"),
+  })
   .argument("[folder]", "convert by folder")
   .action(
-    tryOrExit(async ({ args }) => {
-      const all = Document.findAll({ folderSearch: args.folder });
+    tryOrExit(async ({ args, options }) => {
+      const all = Document.findAll({
+        folderSearch: args.folder,
+        locales: buildLocaleMap(options.locale),
+      });
       for (let doc of all.iter()) {
         if (!doc.isMarkdown) {
           continue;
         }
         const { body: m, attributes: metadata } = fm(doc.rawContent);
-        const h = await m2h(m);
+        const h = await m2h(m, { locale: doc.metadata.locale });
         saveFile(doc.fileInfo.path.replace(/\.md$/, ".html"), h, metadata);
       }
     })

diff --git a/markdown/h2m/handlers/cards.ts b/markdown/h2m/handlers/cards.ts
@@ -1,13 +1,38 @@
+import * as Gettext from "node-gettext";
+import * as fs from "fs";
+import * as path from "path";
+import { DEFAULT_LOCALE } from "../../../libs/constants";
 import { h } from "../h";
 import { asArray } from "../utils";
 import { toText } from "./to-text";
 import { QueryAndTransform } from "./utils";
 
+const gettextLocalizationMap = (() => {
+  const getTextDefaultDomainName = "messages";
+  let gtLocalizationMap = new Map();
+  let localesOnFS = fs
+    .readdirSync(path.join(__dirname, "../../localizations"))
+    .map((str) => str.split(".")[0]);
+  localesOnFS.forEach((localeStr) => {
+    const translations = require("../../localizations/" + localeStr + ".json");
+    const gt = new Gettext();
+    gt.addTranslations(localeStr, getTextDefaultDomainName, translations);
+    gt.setLocale(localeStr);
+    gtLocalizationMap.set(localeStr, gt);
+  });
+  return gtLocalizationMap;
+})();
+
 export const cards: QueryAndTransform[] = [
   ...["note", "warning"].map(
     (className) =>
       [
-        (node) => {
+        (node, { locale = DEFAULT_LOCALE }) => {
+          const defaultLocaleGt = gettextLocalizationMap.get(DEFAULT_LOCALE);
+          let currentLocaleGt = gettextLocalizationMap.get(DEFAULT_LOCALE);
+          if (gettextLocalizationMap.has(locale)) {
+            currentLocaleGt = gettextLocalizationMap.get(locale);
+          }
           if (
             !((node.properties.className as string[]) || []).some(
               (c) => c == className
@@ -25,33 +50,45 @@ export const cards: QueryAndTransform[] = [
           const grandChild = child.children[0];
           return (
             grandChild.tagName == "strong" &&
-            toText(grandChild).toLowerCase() == className + ":"
+            (toText(grandChild) ==
+              currentLocaleGt.gettext("card_" + className + "_label") ||
+              toText(grandChild) ==
+                defaultLocaleGt.gettext("card_" + className + "_label"))
           );
         },
-        (node, t) =>
-          h("blockquote", [
+        (node, t, { locale = DEFAULT_LOCALE }) => {
+          let gt = gettextLocalizationMap.get(DEFAULT_LOCALE);
+          if (gettextLocalizationMap.has(locale)) {
+            gt = gettextLocalizationMap.get(locale);
+          }
+          return h("blockquote", [
             h("paragraph", [
               h("strong", [
-                h(
-                  "text",
-                  className[0].toUpperCase() + className.slice(1) + ":"
-                ),
+                h("text", gt.gettext("card_" + className + "_label")),
               ]),
               ...asArray(t((node.children[0].children as any).slice(1))),
             ]),
             ...asArray(t(node.children.slice(1))),
-          ]),
+          ]);
+        },
       ] as QueryAndTransform
   ),
 
   [
     (node) =>
       node.tagName == "div" &&
       ((node.properties.className as string[]) || "").includes("callout"),
-    (node, t) =>
-      h("blockquote", [
-        h("paragraph", [h("strong", [h("text", "Callout:")])]),
+    (node, t, { locale = DEFAULT_LOCALE }) => {
+      let gt = gettextLocalizationMap.get(DEFAULT_LOCALE);
+      if (gettextLocalizationMap.has(locale)) {
+        gt = gettextLocalizationMap.get(locale);
+      }
+      return h("blockquote", [
+        h("paragraph", [
+          h("strong", [h("text", gt.gettext("card_callout_label"))]),
+        ]),
         ...asArray(t(node.children as any)),
-      ]),
+      ]);
+    },
   ],
 ];
diff --git a/markdown/h2m/index.ts b/markdown/h2m/index.ts
@@ -21,15 +21,18 @@ const getTransformProcessor = (options) =>
       options: { embeddedLanguageFormatting: "off" },
     });
 
-export async function h2m(html, { printAST }: { printAST?: boolean } = {}) {
+export async function h2m(
+  html,
+  { printAST, locale }: { printAST?: boolean; locale?: string } = {}
+) {
   const encodedHTML = encodeKS(html);
   const summary = extractSummary(
     extractSections(cheerio.load(`<div id="_body">${encodedHTML}</div>`))[0]
   );
 
   let unhandled;
   let invalid;
-  const file = await getTransformProcessor({ summary })
+  const file = await getTransformProcessor({ summary, locale })
     .use(() => (result: any) => {
       invalid = result.invalid;
       unhandled = result.unhandled;

diff --git a/markdown/h2m/utils.ts b/markdown/h2m/utils.ts
@@ -9,6 +9,7 @@ export type Options = Partial<{
   shouldWrap: boolean;
   singleLine: boolean;
   summary: string;
+  locale: string;
 }>;
 
 export const asArray = <T extends undefined | unknown | unknown[]>(

diff --git a/markdown/localizations/en-US.json b/markdown/localizations/en-US.json
@@ -0,0 +1,18 @@
+{
+  "translations": {
+    "": {
+      "card_note_label": {
+        "msgid": "card_note_label",
+        "msgstr": ["Note:"]
+      },
+      "card_warning_label": {
+        "msgid": "card_warning_label",
+        "msgstr": ["Warning:"]
+      },
+      "card_callout_label": {
+        "msgid": "card_callout_label",
+        "msgstr": ["Callout:"]
+      }
+    }
+  }
+}
diff --git a/markdown/localizations/fr.json b/markdown/localizations/fr.json
@@ -0,0 +1,18 @@
+{
+  "translations": {
+    "": {
+      "card_note_label": {
+        "msgid": "card_note_label",
+        "msgstr": ["Note :"]
+      },
+      "card_warning_label": {
+        "msgid": "card_warning_label",
+        "msgstr": ["Attention :"]
+      },
+      "card_callout_label": {
+        "msgid": "card_callout_label",
+        "msgstr": ["Remarque :"]
+      }
+    }
+  }
+}
diff --git a/markdown/localizations/ko.json b/markdown/localizations/ko.json
@@ -0,0 +1,18 @@
+{
+  "translations": {
+    "": {
+      "card_note_label": {
+        "msgid": "card_note_label",
+        "msgstr": ["참고:"]
+      },
+      "card_warning_label": {
+        "msgid": "card_warning_label",
+        "msgstr": ["경고:"]
+      },
+      "card_callout_label": {
+        "msgid": "card_callout_label",
+        "msgstr": ["알림:"]
+      }
+    }
+  }
+}