add average frequency tag (#1479)

* added average frequency tag

* fix errors

* add support for term grouping

* added option for average frequency, hides other tags when active

* combines 2 frequencies if one reading is "null"

* updated option-util.test to include average frequency option

* added single space before {

* added average frequency tag

* fix errors

* add support for term grouping

* added option for average frequency, hides other tags when active

* combines 2 frequencies if one reading is "null"

* updated option-util.test to include average frequency option

* added single space before {

* fixes typo, changes averages from object to map

* reformat code, changes display.css to not depend on last-child to hide average frequency

* uses single quotes instead of double quotes

* fixed error that inverts the average frequency setting

* Revert random formatting changes

* Keep for loop const

* Use null coalesce instead of if for reading check

* Split out avg frequency array making

* Set avg freq dict count to 1

* Simplify control flow

* Split out avg frequency data creation

* Clarify merging two readings if one is null

---------

Co-authored-by: kuuuube <[email protected]>

ext/css/display.css

@@ -1977,6 +1977,15 @@ button.footer-notification-close-button {
 :root[data-anki-enabled=false] .action-button[data-action=save-note] {
     display: none;
 }
+
+:root[data-average-frequency=true] .frequency-group-item:not([data-details='Average']) {
+    display: none;
+}
+
+:root[data-average-frequency=false] .frequency-group-item[data-details='Average'] {
+    display: none;
+}
+
 :root[data-audio-enabled=false] .action-button[data-action=play-audio] {
     display: none;
 }

ext/data/schemas/options-schema.json

@@ -102,6 +102,7 @@
                                     "showGuide",
                                     "enableContextMenuScanSelected",
                                     "compactTags",
+                                    "averageFrequency",
                                     "glossaryLayoutMode",
                                     "mainDictionary",
                                     "popupTheme",
@@ -236,6 +237,10 @@
                                         "type": "boolean",
                                         "default": false
                                     },
+                                    "averageFrequency": {
+                                        "type": "boolean",
+                                        "default": false
+                                    },
                                     "glossaryLayoutMode": {
                                         "type": "string",
                                         "enum": ["default", "compact"],

ext/js/dictionary/dictionary-data-util.js

@@ -83,23 +83,108 @@ export function groupTermFrequencies(dictionaryEntry, dictionaryInfo) {
     }
 
     const results = [];
+
+    /** @type {import('dictionary').AverageFrequencyListGroup} */
+    const averages = new Map();
     for (const [dictionary, map2] of map1.entries()) {
+        /** @type {import('dictionary-data-util').TermFrequency[]} */
         const frequencies = [];
         const dictionaryAlias = aliasMap.get(dictionary) ?? dictionary;
         for (const {term, reading, values} of map2.values()) {
-            frequencies.push({
+            const termFrequency = {
                 term,
                 reading,
                 values: [...values.values()],
-            });
+            };
+            frequencies.push(termFrequency);
+
+            const averageFrequencyData = makeAverageFrequencyData(termFrequency, averages.get(term));
+            if (averageFrequencyData) {
+                averages.set(term, averageFrequencyData);
+            }
         }
         const currentDictionaryInfo = dictionaryInfo.find(({title}) => title === dictionary);
         const freqCount = currentDictionaryInfo?.counts?.termMeta.freq ?? 0;
         results.push({dictionary, frequencies, dictionaryAlias, freqCount});
     }
+
+    results.push({dictionary: 'Average', frequencies: makeAverageFrequencyArray(averages), dictionaryAlias: 'Average', freqCount: 1});
+
     return results;
 }
 
+/**
+ * @param {import('dictionary-data-util').TermFrequency} termFrequency
+ * @param {import('dictionary').AverageFrequencyListTerm | undefined} averageTerm
+ * @returns {import('dictionary').AverageFrequencyListTerm | undefined}
+ */
+function makeAverageFrequencyData(termFrequency, averageTerm) {
+    const valuesArray = [...termFrequency.values.values()];
+    const newReading = termFrequency.reading ?? '';
+
+    /** @type {import('dictionary').AverageFrequencyListTerm} */
+    const termMap = typeof averageTerm === 'undefined' ? new Map() : averageTerm;
+
+    const frequencyData = termMap.get(newReading) ?? {currentAvg: 1, count: 0};
+
+    if (valuesArray[0].frequency === null) { return; }
+
+    frequencyData.currentAvg = frequencyData.count / frequencyData.currentAvg + 1 / valuesArray[0].frequency;
+    frequencyData.currentAvg = (frequencyData.count + 1) / frequencyData.currentAvg;
+    frequencyData.count += 1;
+
+    termMap.set(newReading, frequencyData);
+    return termMap;
+}
+
+/**
+ * @param {import('dictionary').AverageFrequencyListGroup} averages
+ * @returns {import('dictionary-data-util').TermFrequency[]}
+ */
+function makeAverageFrequencyArray(averages) {
+    // Merge readings if one is null and there's only two readings
+    // More than one non-null reading cannot be merged since it cannot be determined which reading to merge with
+    for (const currentTerm of averages.keys()) {
+        const readingsMap = averages.get(currentTerm);
+        if (!readingsMap) { continue; } // Skip if readingsMap is undefined
+
+        const readingArray = [...readingsMap.keys()];
+        const nullIndex = readingArray.indexOf('');
+
+        if (readingArray.length === 2 && nullIndex >= 0) {
+            const key1 = readingArray[0];
+            const key2 = readingArray[1];
+
+            const value1 = readingsMap.get(key1);
+            const value2 = readingsMap.get(key2);
+
+            if (!value1 || !value2) { continue; } // Skip if any value is undefined
+
+            const avg1 = value1.currentAvg;
+            const count1 = value1.count;
+            const avg2 = value2.currentAvg;
+            const count2 = value2.count;
+
+            const newcount = count1 + count2;
+            const newavg = newcount / (count1 / avg1 + count2 / avg2);
+
+            const validKey = nullIndex === 0 ? key2 : key1;
+            readingsMap.set(validKey, {currentAvg: newavg, count: newcount});
+            readingsMap.delete('');
+        }
+    }
+
+    // Convert averages Map back to array format
+    return [...averages.entries()].flatMap(([termName, termMap]) => [...termMap.entries()].map(([readingName, data]) => ({
+        term: termName,
+        reading: readingName,
+        values: [{
+            frequency: Math.round(data.currentAvg),
+            displayValue: Math.round(data.currentAvg).toString(),
+        }],
+    })));
+}
+
 /**
  * @param {import('dictionary').KanjiFrequency[]} sourceFrequencies
  * @param {import('dictionary-importer').Summary[]} dictionaryInfo

ext/js/display/display.js

@@ -1202,6 +1202,7 @@ export class Display extends EventDispatcher {
         data.resultOutputMode = `${options.general.resultOutputMode}`;
         data.glossaryLayoutMode = `${options.general.glossaryLayoutMode}`;
         data.compactTags = `${options.general.compactTags}`;
+        data.averageFrequency = `${options.general.averageFrequency}`;
         data.frequencyDisplayMode = `${options.general.frequencyDisplayMode}`;
         data.termDisplayMode = `${options.general.termDisplayMode}`;
         data.enableSearchTags = `${options.scanning.enableSearchTags}`;

ext/settings.html

@@ -1090,6 +1090,15 @@ <h1>Yomitan Settings</h1>
                 </div></div>
             </div>
         </div>
+        <div class="settings-item"><div class="settings-item-inner">
+            <div class="settings-item-left">
+                <div class="settings-item-label">Average frequencies</div>
+                <div class="settings-item-description">Compress frequency tags into one "Average" frequency tag based on the harmonic mean.</div>
+            </div>
+            <div class="settings-item-right">
+                <label class="toggle"><input type="checkbox" data-setting="general.averageFrequency"><span class="toggle-body"><span class="toggle-track"></span><span class="toggle-knob"></span></span></label>
+            </div>
+        </div></div>
     </div>
 
     <!-- Popup Position & Size -->

test/options-util.test.js

@@ -56,6 +56,7 @@ function createProfileOptionsTestData1() {
             popupScaleRelativeToVisualViewport: true,
             showGuide: true,
             compactTags: false,
+            averageFrequency: false,
             compactGlossaries: false,
             mainDictionary: '',
             popupTheme: 'default',
@@ -288,6 +289,7 @@ function createProfileOptionsUpdatedTestData1() {
             showGuide: true,
             enableContextMenuScanSelected: true,
             compactTags: false,
+            averageFrequency: false,
             glossaryLayoutMode: 'default',
             mainDictionary: '',
             popupTheme: 'light',

types/ext/dictionary.d.ts

@@ -532,3 +532,27 @@ export type TermSource = {
      */
     isPrimary: boolean;
 };
+
+/**
+ * Dictionaries containing the harmonic mean frequency of specific term-reading pairs.
+ */
+export type AverageFrequencyListGroup = Map<string, AverageFrequencyListTerm>;
+
+/**
+ * Contains the average frequency of a term, with all its readings.
+ */
+export type AverageFrequencyListTerm = Map<string, AverageFrequencyListReading>;
+
+/**
+ * The number of dictionary frequencies used to compute the average.
+ */
+export type AverageFrequencyListReading = {
+    /**
+     * The current average frequency.
+     */
+    currentAvg: number;
+    /**
+     * The number of dictionary frequencies used to compute the average.
+     */
+    count: number;
+};

types/ext/settings.d.ts

@@ -128,6 +128,7 @@ export type GeneralOptions = {
     showGuide: boolean;
     enableContextMenuScanSelected: boolean;
     compactTags: boolean;
+    averageFrequency: boolean;
     glossaryLayoutMode: GlossaryLayoutMode;
     mainDictionary: string;
     popupTheme: PopupTheme;