reorganizing

yomidevs · Jan 20, 2024 · 87660d7 · 87660d7
1 parent 1d16219
commit 87660d7
Show file tree

Hide file tree

Showing 10 changed files with 47 additions and 905 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 *.json
 !tag_bank_term.json
 !tag_bank_ipa.json
+!languages.json
 !package.json
 
 *.zip

diff --git a/1-create-folders.js b/1-create-folders.js
@@ -17,16 +17,14 @@
 const {mkdirSync} = require('fs');
 
 const folders = [
-    'freq',
-    'freq/metadata',
     'kaikki',
     'sentences',
     'tidy',
     'language',
     'temp',
     'temp/dict',
-    'temp/freq',
-    'temp/ipa'
+    'temp/ipa',
+    'test'
 ];
 
 for (const folder of folders) {

diff --git a/5-make-yezichak.js → 3-make-yomitan.js b/5-make-yezichak.js → 3-make-yomitan.js
@@ -23,9 +23,9 @@ const {source_iso, target_iso, DEBUG_WORD, DICT_NAME} = process.env;
 
 const currentDate = date.format(now, 'YYYY.MM.DD');
 
-console.log(`5-make-yezichak.js: reading lemmas`);
+console.log(`3-make-yomitan.js: reading lemmas`);
 const lemmaDict = JSON.parse(readFileSync(`data/tidy/${source_iso}-${target_iso}-lemmas.json`));
-console.log(`5-make-yezichak.js: reading forms`);
+console.log(`3-make-yomitan.js: reading forms`);
 const formDict = JSON.parse(readFileSync(`data/tidy/${source_iso}-${target_iso}-forms.json`));
 
 if (!existsSync(`data/language/${source_iso}/${target_iso}`)) {
@@ -112,7 +112,7 @@ const skippedTermTags = {};
 let ipaCount = 0;
 let termTagCount = 0;
 
-console.log('5-make-yezichak.js: processing lemmas...');
+console.log('3-make-yomitan.js: processing lemmas...');
 for (const [lemma, infoMap] of Object.entries(lemmaDict)) {
     normalizedLemma = normalizeOrthography(lemma);
 
@@ -258,7 +258,7 @@ const multiwordInflections = [
     'female equivalent', // de
 ];
 
-console.log('5-make-yezichak.js: Processing forms...');
+console.log('3-make-yomitan.js: Processing forms...');
 for (const [form, allInfo] of Object.entries(formDict)) {
     for (const [lemma, info] of Object.entries(allInfo)) {
         for (const [pos, glosses] of Object.entries(info)) {
@@ -349,7 +349,7 @@ const indexJson = {
 const folders = ['dict', 'ipa'];
 
 for (const folder of folders) {
-    console.log(`5-make-yezichak.js: Writing ${folder}...`);
+    console.log(`3-make-yomitan.js: Writing ${folder}...`);
     for (const file of readdirSync(`${tempPath}/${folder}`)) {
         if (file.includes('term_')) { unlinkSync(`${tempPath}/${folder}/${file}`); }
     }
@@ -372,7 +372,7 @@ writeFileSync(`data/language/${source_iso}/${target_iso}/skippedIpaTags.json`, J
 console.log('total term tags', termTagCount, 'skipped term tags', Object.values(skippedTermTags).reduce((a, b) => a + (parseInt(b) || 0), 0));
 writeFileSync(`data/language/${source_iso}/${target_iso}/skippedTermTags.json`, JSON.stringify(sortBreakdown(skippedTermTags), null, 2));
 
-console.log('5-make-yezichak.js: Done!');
+console.log('3-make-yomitan.js: Done!');
 
 function writeInBatches(inputArray, filenamePrefix, batchSize = 100000) {
     console.log(`Writing ${inputArray.length.toLocaleString()} entries...`);