Moving lexicon loading to a web worker.

wiktionary_pron/scripts/lexicon.js

@@ -1,4 +1,4 @@
-import { loadFileFromZip, splitAndAppend } from "./utils.js";
+import { loadFileFromZip } from "./utils.js";
 
 async function loadLexicon(language) {
   const languages = {
@@ -11,33 +11,15 @@ async function loadLexicon(language) {
   );
 
   function process_lexicon(text) {
-    const split = text.split(/\r?\n/);
-    const lines = [];
+    return new Promise((resolve) => {
+      const worker = new Worker("scripts/lexicon_loader_worker.js");
 
-    const chunk = 10000;
-    let index = 0;
+      worker.onmessage = function (e) {
+        resolve(e.data);
+      };
 
-    function processChunk() {
-      return new Promise((resolve) => {
-        let cnt = chunk;
-        while (cnt-- && index < split.length) {
-          const [text, ipa] = splitAndAppend(split[index], "\t", 1);
-          if (!ipa.includes("|")) {
-            lines.push([text, ipa.split(" ").join("")]);
-          }
-          index++;
-        }
-        if (index < split.length) {
-          // simulate async iteration
-          setTimeout(() => resolve(processChunk()), 1);
-        } else {
-          const dict = new Map(lines.reverse());
-          resolve(dict);
-        }
-      });
-    }
-
-    return processChunk();
+      worker.postMessage(text);
+    });
   }
 
   const lexicon = await process_lexicon(wordPairsList);

wiktionary_pron/scripts/lexicon_loader_worker.js

@@ -0,0 +1,16 @@
+self.onmessage = function (e) {
+  const splitAndAppend = (str, delim, count) => {
+    const index = str.indexOf(delim);
+    return [str.slice(0, index), str.slice(index + 1)];
+  };
+  const split = e.data.split(/\r?\n/);
+  const lines = [];
+
+  for (let index = 0; index < split.length; index++) {
+    const [text, ipa] = splitAndAppend(split[index], "\t", 1);
+    lines.push([text, ipa.split(" ").join("")]);
+  }
+
+  const dict = new Map(lines.reverse());
+  self.postMessage(dict);
+};

wiktionary_pron/scripts/utils.js

@@ -1,8 +1,3 @@
-const splitAndAppend = (str, delim, count) => {
-  const arr = str.split(delim);
-  return [...arr.splice(0, count), arr.join(delim)];
-};
-
 async function asyncMapStrict(arr, fn) {
   const result = [];
   // console.time("Elapsed time :");
@@ -164,11 +159,11 @@ function get_ipa_no_cache(text, args) {
     case "German":
       if (langForm === "Phonemic") {
         let dictRecord = globalThis.lexicon.get(
-            cleanText.replace(/[^\p{Letter}\p{Mark}-]+/gu, ""),
+          cleanText.replace(/[^\p{Letter}\p{Mark}-]+/gu, ""),
         );
         if (!dictRecord) {
           dictRecord = globalThis.lexicon.get(
-              cleanText.replace(/[^\p{Letter}\p{Mark}-]+/gu, "").toLowerCase(),
+            cleanText.replace(/[^\p{Letter}\p{Mark}-]+/gu, "").toLowerCase(),
           );
         }
         console.log(cleanText, dictRecord);
@@ -330,7 +325,6 @@ export {
   memoizeLocalStorage,
   macronize,
   loadJs,
-  splitAndAppend,
   loadFileFromZip,
   createElementFromHTML,
 };