diff --git a/app/build.gradle b/app/build.gradle index 7c72366ce..6c2cfee1d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -31,8 +31,8 @@ android { applicationId "com.kazumaproject.markdownhelperkeyboard" minSdk 24 targetSdk 36 - versionCode 799 - versionName "1.7.106" + versionCode 800 + versionName "1.7.107" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } diff --git a/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/EnglishReadingDictionaryInstrumentedTest.kt b/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/EnglishReadingDictionaryInstrumentedTest.kt new file mode 100644 index 000000000..47d0bba86 --- /dev/null +++ b/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/EnglishReadingDictionaryInstrumentedTest.kt @@ -0,0 +1,108 @@ +package com.kazumaproject.markdownhelperkeyboard.converter + +import android.content.Context +import androidx.preference.PreferenceManager +import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.kazumaproject.markdownhelperkeyboard.converter.engine.KanaKanjiEngine +import com.kazumaproject.markdownhelperkeyboard.dictionary_override.DictionaryOverrideStore +import com.kazumaproject.markdownhelperkeyboard.ime_service.di.KanaKanjiEngineEntryPoint +import dagger.hilt.android.EntryPointAccessors +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class EnglishReadingDictionaryInstrumentedTest { + + @Test + fun englishReadingCandidatesFollowTheSetting() = runBlocking { + val context = ApplicationProvider.getApplicationContext() + val preferences = PreferenceManager.getDefaultSharedPreferences(context) + val hadPreviousValue = preferences.contains( + DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE + ) + val previousValue = preferences.getBoolean( + DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE, + true, + ) + val entryPoint = EntryPointAccessors.fromApplication( + context.applicationContext, + KanaKanjiEngineEntryPoint::class.java, + ) + val engine = entryPoint.kanaKanjiEngine() + val repository = entryPoint.userDictionaryRepository() + + try { + setEnabled(preferences, true) + engine.applyDictionaryOverrideState(context) + val enabledCandidates = convert(engine, repository, "かー", nBest = 4) + val enabledEnglishCases = enabledCandidates.map { it.string }.toSet() + assertTrue( + "English reading case candidates were not found: $enabledCandidates", + setOf("car", "Car", "CAR").all(enabledEnglishCases::contains), + ) + val galleryCandidates = convert(engine, repository, "ぎゃらりー") + assertTrue( + "Gallery candidate was not found: $galleryCandidates", + galleryCandidates.any { it.string == "gallery" }, + ) + val artGalleryCandidates = convert(engine, repository, "あーとぎゃらりー") + assertTrue( + "Art gallery candidate was not found: $artGalleryCandidates", + artGalleryCandidates.any { it.string == "art gallery" }, + ) + + setEnabled(preferences, false) + engine.applyDictionaryOverrideState(context) + val disabledCandidates = convert(engine, repository, "かー") + assertFalse( + "English reading case candidates remained after disabling: $disabledCandidates", + disabledCandidates.any { it.string in setOf("car", "Car", "CAR") }, + ) + } finally { + if (hadPreviousValue) { + setEnabled(preferences, previousValue) + } else { + preferences.edit() + .remove(DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE) + .commit() + } + engine.applyDictionaryOverrideState(context) + } + } + + private suspend fun convert( + engine: KanaKanjiEngine, + repository: com.kazumaproject.markdownhelperkeyboard.repository.UserDictionaryRepository, + input: String, + nBest: Int = 64, + ) = engine.getCandidatesWithBunsetsuSeparation( + input = input, + n = nBest, + mozcUtPersonName = false, + mozcUTPlaces = false, + mozcUTWiki = false, + mozcUTNeologd = false, + mozcUTWeb = false, + userDictionaryRepository = repository, + learnRepository = null, + isOmissionSearchEnable = false, + enableTypoCorrectionJapaneseFlick = false, + enableTypoCorrectionQwertyEnglish = false, + typoCorrectionOffsetScore = 3000, + omissionSearchOffsetScore = 1900, + beamWidth = 20, + ).candidates + + private fun setEnabled( + preferences: android.content.SharedPreferences, + enabled: Boolean, + ) { + preferences.edit() + .putBoolean(DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE, enabled) + .commit() + } +} diff --git a/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/EnglishReadingDictionaryPerformanceInstrumentedTest.kt b/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/EnglishReadingDictionaryPerformanceInstrumentedTest.kt new file mode 100644 index 000000000..929fc876b --- /dev/null +++ b/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/EnglishReadingDictionaryPerformanceInstrumentedTest.kt @@ -0,0 +1,249 @@ +package com.kazumaproject.markdownhelperkeyboard.converter + +import android.content.Context +import android.os.Debug +import androidx.preference.PreferenceManager +import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.kazumaproject.markdownhelperkeyboard.converter.candidate.Candidate +import com.kazumaproject.markdownhelperkeyboard.converter.engine.KanaKanjiEngine +import com.kazumaproject.markdownhelperkeyboard.dictionary_override.DictionaryOverrideStore +import com.kazumaproject.markdownhelperkeyboard.ime_service.di.KanaKanjiEngineEntryPoint +import dagger.hilt.android.EntryPointAccessors +import kotlinx.coroutines.runBlocking +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File +import kotlin.math.ceil +import kotlin.system.measureNanoTime + +@RunWith(AndroidJUnit4::class) +class EnglishReadingDictionaryPerformanceInstrumentedTest { + + @Test + fun measureEnabledAndDisabledConversion() = runBlocking { + val context = ApplicationProvider.getApplicationContext() + val preferences = PreferenceManager.getDefaultSharedPreferences(context) + val hadPreviousValue = preferences.contains( + DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE + ) + val previousValue = preferences.getBoolean( + DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE, + true, + ) + val entryPoint = EntryPointAccessors.fromApplication( + context.applicationContext, + KanaKanjiEngineEntryPoint::class.java, + ) + val engine = entryPoint.kanaKanjiEngine() + val repository = entryPoint.userDictionaryRepository() + + try { + val enabled = measureState( + context = context, + preferences = preferences, + engine = engine, + repository = repository, + enabled = true, + ) + val disabled = measureState( + context = context, + preferences = preferences, + engine = engine, + repository = repository, + enabled = false, + ) + val report = buildString { + appendLine("device=${android.os.Build.MANUFACTURER} ${android.os.Build.MODEL} API ${android.os.Build.VERSION.SDK_INT}") + appendLine("input=かー") + appendLine("warmups=$WARMUPS") + appendLine("iterations=$ITERATIONS") + appendLine(enabled.toReport("enabled")) + appendLine(disabled.toReport("disabled")) + appendLine("warmP50DeltaMs=${enabled.p50Ms - disabled.p50Ms}") + appendLine("warmP95DeltaMs=${enabled.p95Ms - disabled.p95Ms}") + appendLine("retainedHeapDeltaBytes=${enabled.retainedHeapBytes - disabled.retainedHeapBytes}") + appendLine("nativeRetainedDeltaBytes=${enabled.nativeRetainedBytes - disabled.nativeRetainedBytes}") + appendLine("enabledEnglishCandidateIndexes=${ENGLISH_CASES.associateWith { word -> enabled.candidates.indexOfFirst { it.string == word } }}") + appendLine("enabledEnglishCandidateScores=${ENGLISH_CASES.associateWith { word -> enabled.candidates.firstOrNull { it.string == word }?.score }}") + appendLine("enabledNBest4Candidates=${enabled.nBestCandidates.joinToString("|") { it.string }}") + appendLine("enabledCandidates=${enabled.candidates.take(16).joinToString("|") { it.string }}") + appendLine("disabledCandidates=${disabled.candidates.take(16).joinToString("|") { it.string }}") + } + File(context.filesDir, "conversion-perf").apply { mkdirs() } + .resolve("english-reading.txt") + .writeText(report) + println(report) + } finally { + if (hadPreviousValue) { + setEnabled(preferences, previousValue) + } else { + preferences.edit() + .remove(DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE) + .commit() + } + engine.applyDictionaryOverrideState(context) + } + } + + private suspend fun measureState( + context: Context, + preferences: android.content.SharedPreferences, + engine: KanaKanjiEngine, + repository: com.kazumaproject.markdownhelperkeyboard.repository.UserDictionaryRepository, + enabled: Boolean, + ): Measurement { + setEnabled(preferences, enabled) + val applyMs = measureNanoTime { engine.applyDictionaryOverrideState(context) } / 1_000_000.0 + repeat(WARMUPS) { convert(engine, repository, nBest = 4) } + forceGc() + + val heapBefore = usedHeapBytes() + val nativeBefore = Debug.getNativeHeapAllocatedSize() + val allocatedBefore = runtimeStat("art.gc.bytes-allocated") + val gcBefore = runtimeStat("art.gc.gc-count") + val samples = DoubleArray(ITERATIONS) + repeat(ITERATIONS) { index -> + samples[index] = measureNanoTime { + convert(engine, repository, nBest = 4) + } / 1_000_000.0 + } + val allocatedAfter = runtimeStat("art.gc.bytes-allocated") + val gcAfter = runtimeStat("art.gc.gc-count") + val heapAfter = usedHeapBytes() + val nativeAfter = Debug.getNativeHeapAllocatedSize() + forceGc() + val heapAfterGc = usedHeapBytes() + val nativeAfterGc = Debug.getNativeHeapAllocatedSize() + + val validationCandidates = convert(engine, repository, nBest = 64) + val nBestCandidates = convert(engine, repository, nBest = 4) + if (enabled) { + check(ENGLISH_CASES.all { word -> validationCandidates.any { it.string == word } }) { + "Enabled dictionary did not produce all case candidates: $validationCandidates" + } + check(ENGLISH_CASES.all { word -> nBestCandidates.any { it.string == word } }) { + "Enabled dictionary did not produce all case candidates in n-best candidates: $nBestCandidates" + } + } else { + check(validationCandidates.none { it.string in ENGLISH_CASES }) { + "Disabled dictionary still produced English case candidates: $validationCandidates" + } + check(nBestCandidates.none { it.string in ENGLISH_CASES }) { + "Disabled dictionary still produced English case candidates in n-best candidates: $nBestCandidates" + } + } + return Measurement( + enabled = enabled, + applyMs = applyMs, + avgMs = samples.average(), + p50Ms = percentile(samples, 0.50), + p95Ms = percentile(samples, 0.95), + maxMs = samples.maxOrNull() ?: 0.0, + allocatedBytesPerConversion = (allocatedAfter - allocatedBefore) / ITERATIONS, + gcCount = gcAfter - gcBefore, + heapBeforeBytes = heapBefore, + heapAfterBytes = heapAfter, + retainedHeapBytes = heapAfterGc - heapBefore, + nativeBeforeBytes = nativeBefore, + nativeAfterBytes = nativeAfter, + nativeRetainedBytes = nativeAfterGc - nativeBefore, + candidates = validationCandidates, + nBestCandidates = nBestCandidates, + ) + } + + private suspend fun convert( + engine: KanaKanjiEngine, + repository: com.kazumaproject.markdownhelperkeyboard.repository.UserDictionaryRepository, + nBest: Int, + ): List = engine.getCandidatesWithBunsetsuSeparation( + input = "かー", + n = nBest, + mozcUtPersonName = false, + mozcUTPlaces = false, + mozcUTWiki = false, + mozcUTNeologd = false, + mozcUTWeb = false, + userDictionaryRepository = repository, + learnRepository = null, + isOmissionSearchEnable = false, + enableTypoCorrectionJapaneseFlick = false, + enableTypoCorrectionQwertyEnglish = false, + typoCorrectionOffsetScore = 3000, + omissionSearchOffsetScore = 1900, + beamWidth = 20, + ).candidates + + private fun setEnabled( + preferences: android.content.SharedPreferences, + enabled: Boolean, + ) { + preferences.edit() + .putBoolean(DictionaryOverrideStore.ENGLISH_READING_ENABLED_PREFERENCE, enabled) + .commit() + } + + private fun forceGc() { + repeat(3) { + Runtime.getRuntime().gc() + System.runFinalization() + Thread.sleep(100) + } + } + + private fun usedHeapBytes(): Long = Runtime.getRuntime().run { + totalMemory() - freeMemory() + } + + private fun runtimeStat(name: String): Long = + Debug.getRuntimeStat(name)?.toLongOrNull() ?: 0L + + private fun percentile(values: DoubleArray, fraction: Double): Double { + val sorted = values.sorted() + return sorted[(ceil(sorted.size * fraction).toInt() - 1).coerceIn(sorted.indices)] + } + + private data class Measurement( + val enabled: Boolean, + val applyMs: Double, + val avgMs: Double, + val p50Ms: Double, + val p95Ms: Double, + val maxMs: Double, + val allocatedBytesPerConversion: Long, + val gcCount: Long, + val heapBeforeBytes: Long, + val heapAfterBytes: Long, + val retainedHeapBytes: Long, + val nativeBeforeBytes: Long, + val nativeAfterBytes: Long, + val nativeRetainedBytes: Long, + val candidates: List, + val nBestCandidates: List, + ) { + fun toReport(label: String): String = buildString { + appendLine("[$label]") + appendLine("enabled=$enabled") + appendLine("applyMs=$applyMs") + appendLine("avgMs=$avgMs") + appendLine("p50Ms=$p50Ms") + appendLine("p95Ms=$p95Ms") + appendLine("maxMs=$maxMs") + appendLine("allocatedBytesPerConversion=$allocatedBytesPerConversion") + appendLine("gcCount=$gcCount") + appendLine("heapBeforeBytes=$heapBeforeBytes") + appendLine("heapAfterBytes=$heapAfterBytes") + appendLine("retainedHeapBytes=$retainedHeapBytes") + appendLine("nativeBeforeBytes=$nativeBeforeBytes") + appendLine("nativeAfterBytes=$nativeAfterBytes") + appendLine("nativeRetainedBytes=$nativeRetainedBytes") + }.trimEnd() + } + + private companion object { + const val WARMUPS = 10 + const val ITERATIONS = 50 + val ENGLISH_CASES = setOf("car", "Car", "CAR") + } +} diff --git a/app/src/main/assets/english_reading/tango.dat.zip b/app/src/main/assets/english_reading/tango.dat.zip new file mode 100644 index 000000000..03e1a8fef Binary files /dev/null and b/app/src/main/assets/english_reading/tango.dat.zip differ diff --git a/app/src/main/assets/english_reading/token.dat.zip b/app/src/main/assets/english_reading/token.dat.zip new file mode 100644 index 000000000..4946d2f09 Binary files /dev/null and b/app/src/main/assets/english_reading/token.dat.zip differ diff --git a/app/src/main/assets/english_reading/yomi.dat.zip b/app/src/main/assets/english_reading/yomi.dat.zip new file mode 100644 index 000000000..fe7cc97f6 Binary files /dev/null and b/app/src/main/assets/english_reading/yomi.dat.zip differ diff --git a/app/src/main/assets/ngram/system_ngram.dat b/app/src/main/assets/ngram/system_ngram.dat index 832c9bdc8..e21650bfe 100644 Binary files a/app/src/main/assets/ngram/system_ngram.dat and b/app/src/main/assets/ngram/system_ngram.dat differ diff --git a/app/src/main/assets/system/token.dat.zip b/app/src/main/assets/system/token.dat.zip index 6a9c6fb56..e1cf7b063 100644 Binary files a/app/src/main/assets/system/token.dat.zip and b/app/src/main/assets/system/token.dat.zip differ diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/engine/KanaKanjiEngine.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/engine/KanaKanjiEngine.kt index 3231b31b1..37875afa8 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/engine/KanaKanjiEngine.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/engine/KanaKanjiEngine.kt @@ -83,6 +83,8 @@ private const val POS_ID_COUNTER_TIME: Short = 2015 private const val POS_ID_NUMBER_ARABIC: Short = 2044 private const val POS_ID_NUMBER_SEPARATED: Short = 2045 private const val POS_ID_NUMBER_KANJI: Short = 2046 +private const val ENGLISH_READING_CAPITALIZED_SCORE_OFFSET = 1_500 +private const val ENGLISH_READING_UPPERCASE_SCORE_OFFSET = 3_000 internal fun shouldIncludeSymbolEmojiReading( inputLength: Int, @@ -247,6 +249,9 @@ class KanaKanjiEngine { @Volatile private var kotowazaDictionaryEnabled: Boolean = true + @Volatile + private var englishReadingDictionary: TripleDictionaryData? = null + private var personYomiTrie: LOUDSWithTermId? = null private var personTangoTrie: LOUDS? = null private var personTokenArray: TokenArray? = null @@ -463,6 +468,7 @@ class KanaKanjiEngine { val newEmoji = loadTripleDictionary(reader, DictionaryCategory.EMOJI) val newEmoticon = loadTripleDictionary(reader, DictionaryCategory.EMOTICON) val newSymbol = loadTripleDictionary(reader, DictionaryCategory.SYMBOL) + val newEnglishReading = loadEnglishReadingDictionary(reader) val readingCorrectionState = reader.resolveCategoryLoadState(DictionaryCategory.READING_CORRECTION) @@ -494,6 +500,7 @@ class KanaKanjiEngine { assignEmojiDictionary(newEmoji) assignEmoticonDictionary(newEmoticon) assignSymbolDictionary(newSymbol) + assignEnglishReadingDictionary(newEnglishReading) if (newReadingCorrection != null) { assignReadingCorrectionDictionary(newReadingCorrection) @@ -533,7 +540,9 @@ class KanaKanjiEngine { val newWiki = loadOptionalTripleDictionary(reader, DictionaryCategory.WIKI) val newNeologd = loadOptionalTripleDictionary(reader, DictionaryCategory.NEOLOGD) val newWeb = loadOptionalTripleDictionary(reader, DictionaryCategory.WEB) + val newEnglishReading = loadEnglishReadingDictionary(reader) synchronized(this) { + assignEnglishReadingDictionary(newEnglishReading) assignPersonDictionary(newPerson) assignPlacesDictionary(newPlaces) assignWikiDictionary(newWiki) @@ -556,6 +565,17 @@ class KanaKanjiEngine { null } + private fun loadEnglishReadingDictionary( + reader: DictionaryBinaryReader, + ): TripleDictionaryData? = + if (reader.resolveCategoryLoadState(DictionaryCategory.ENGLISH_READING) == + DictionaryCategoryLoadState.Disabled + ) { + null + } else { + loadTripleDictionary(reader, DictionaryCategory.ENGLISH_READING) + } + private fun loadTripleDictionary( reader: DictionaryBinaryReader, category: DictionaryCategory, @@ -818,6 +838,20 @@ class KanaKanjiEngine { kotowazaSuccinctBitVectorTangoLBS = data.succinctBitVectorTangoLBS } + private fun assignEnglishReadingDictionary(data: TripleDictionaryData?) { + englishReadingDictionary = data + if (!::graphBuilder.isInitialized) return + graphBuilder.updateEnglishReadingDictionary( + yomiTrie = data?.yomiTrie, + tangoTrie = data?.tangoTrie, + tokenArray = data?.tokenArray, + succinctBitVectorLBSYomi = data?.succinctBitVectorLBSYomi, + succinctBitVectorIsLeafYomi = data?.succinctBitVectorIsLeafYomi, + succinctBitVectorTokenArray = data?.succinctBitVectorTokenArray, + succinctBitVectorTangoLBS = data?.succinctBitVectorTangoLBS, + ) + } + private fun assignPersonDictionary(data: TripleDictionaryData?) { personTangoTrie = data?.tangoTrie personYomiTrie = data?.yomiTrie @@ -1593,7 +1627,8 @@ class KanaKanjiEngine { val resultListFinal = resultList.sortedWith(compareBy { it.score }.thenBy { it.string }) - return resultListFinal + kotowazaListDeferred + symbolHalfWidthListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred + val englishReadingDeferred = deferredEnglishReadingCandidates(input, resultList) + return resultListFinal + englishReadingDeferred + kotowazaListDeferred + symbolHalfWidthListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred } @@ -2129,7 +2164,7 @@ class KanaKanjiEngine { compareByDescending { it.string in systemNgramMatchedCandidates } .thenBy { it.score } .thenBy { it.string }, - ) + kotowazaListDeferred + symbolHalfWidthListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred + ) + deferredEnglishReadingCandidates(input, resultList) + kotowazaListDeferred + symbolHalfWidthListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred return BunsetsuCandidateResult( candidates = resultListFinal, @@ -2655,7 +2690,7 @@ class KanaKanjiEngine { ) val finalList = - resultListFinal + kotowazaListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + hirakanaAndKana + yomiPartListDeferred + symbolListDeferred + singleKanjiListDeferred + resultListFinal + deferredEnglishReadingCandidates(input, resultList) + kotowazaListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + hirakanaAndKana + yomiPartListDeferred + symbolListDeferred + singleKanjiListDeferred return BunsetsuCandidateResult( candidates = finalList, @@ -3172,7 +3207,7 @@ class KanaKanjiEngine { val resultListFinal = resultList.sortedWith(compareBy { it.score }.thenBy { it.string }) - return resultListFinal + kotowazaListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + hirakanaAndKana + yomiPartListDeferred + symbolListDeferred + singleKanjiListDeferred + return resultListFinal + deferredEnglishReadingCandidates(input, resultList) + kotowazaListDeferred + (englishDeferred + englishZenkaku).sortedBy { it.score } + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + hirakanaAndKana + yomiPartListDeferred + symbolListDeferred + singleKanjiListDeferred } @@ -3672,7 +3707,7 @@ class KanaKanjiEngine { val resultListFinal = resultList.sortedWith(compareBy { it.score }.thenBy { it.string }) - return resultListFinal + (englishDeferred + englishZenkaku).sortedBy { it.score } + symbolHalfWidthListDeferred + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + kotowazaListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred + return resultListFinal + deferredEnglishReadingCandidates(input, resultList) + (englishDeferred + englishZenkaku).sortedBy { it.score } + symbolHalfWidthListDeferred + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + kotowazaListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred } @@ -4197,7 +4232,7 @@ class KanaKanjiEngine { compareByDescending { it.string in systemNgramMatchedCandidates } .thenBy { it.score } .thenBy { it.string }, - ) + (englishDeferred + englishZenkaku).sortedBy { it.score } + symbolHalfWidthListDeferred + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + kotowazaListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred + ) + deferredEnglishReadingCandidates(input, resultList) + (englishDeferred + englishZenkaku).sortedBy { it.score } + symbolHalfWidthListDeferred + (emojiListDeferred + emoticonListDeferred).sortedBy { it.score } + symbolListDeferred + kotowazaListDeferred + hirakanaAndKana + yomiPartListDeferred + singleKanjiListDeferred return BunsetsuCandidateResult( candidates = resultListFinal, @@ -5003,6 +5038,74 @@ class KanaKanjiEngine { } } + private fun deferredEnglishReadingCandidates( + input: String, + existingCandidates: Iterable, + ): List { + val dictionary = englishReadingDictionary ?: return emptyList() + val nodeIndex = dictionary.yomiTrie.getNodeIndex( + input, + dictionary.succinctBitVectorLBSYomi, + ) + if (nodeIndex <= 0) return emptyList() + val termId = dictionary.yomiTrie.getTermIdShortArray( + nodeIndex, + dictionary.succinctBitVectorIsLeafYomi, + ) + val existingStrings = existingCandidates.asSequence().map { it.string }.toMutableSet() + return buildList { + fun addCandidate(candidate: Candidate) { + if (existingStrings.add(candidate.string)) add(candidate) + } + + fun addEnglishCaseCandidates(candidate: Candidate) { + addCandidate(candidate) + if (candidate.string.none { it in 'A'..'Z' || it in 'a'..'z' }) return + + val lowercase = candidate.string.lowercase(Locale.ROOT) + val capitalized = lowercase.replaceFirstChar { char -> + char.uppercase(Locale.ROOT) + } + val uppercase = lowercase.uppercase(Locale.ROOT) + addCandidate(candidate.copy(string = lowercase)) + addCandidate( + candidate.copy( + string = capitalized, + score = candidate.score + ENGLISH_READING_CAPITALIZED_SCORE_OFFSET, + ) + ) + addCandidate( + candidate.copy( + string = uppercase, + score = candidate.score + ENGLISH_READING_UPPERCASE_SCORE_OFFSET, + ) + ) + } + + dictionary.tokenArray.forEachDictionaryByYomiTermIdShortArray( + termId, + dictionary.succinctBitVectorTokenArray, + ) { posTableIndex, wordCost, nodeId -> + val candidate = Candidate( + string = when (nodeId) { + -2 -> input + -1 -> input.hiraToKata() + else -> dictionary.tangoTrie.getLetterShortArray( + nodeId, + dictionary.succinctBitVectorTangoLBS, + ) + }, + type = 2, + length = input.length.toUByte(), + score = wordCost.toInt(), + leftId = dictionary.tokenArray.leftIds[posTableIndex.toInt()], + rightId = dictionary.tokenArray.rightIds[posTableIndex.toInt()], + ) + addEnglishCaseCandidates(candidate) + } + } + } + private fun deferredFromDictionarySingleKanji( input: String, yomiTrie: LOUDSWithTermId, diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/graph/GraphBuilder.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/graph/GraphBuilder.kt index b8b7fd271..7bf059d8c 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/graph/GraphBuilder.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/graph/GraphBuilder.kt @@ -31,6 +31,7 @@ class GraphBuilder { val systemTypoProgress: Map, val systemUserTypoProgress: Map, val systemPrefixStates: Map, + val englishReadingPrefixStates: Map, val systemUserPrefixStates: Map, val wikiPrefixStates: Map, val webPrefixStates: Map, @@ -176,6 +177,14 @@ class GraphBuilder { private var systemUserSuccinctBitVectorTokenArray: SuccinctBitVector? = null private var systemUserSuccinctBitVectorTangoLBS: SuccinctBitVector? = null + private var englishReadingYomiTrie: LOUDSWithTermId? = null + private var englishReadingTangoTrie: LOUDS? = null + private var englishReadingTokenArray: TokenArray? = null + private var englishReadingSuccinctBitVectorLBSYomi: SuccinctBitVector? = null + private var englishReadingSuccinctBitVectorIsLeafYomi: SuccinctBitVector? = null + private var englishReadingSuccinctBitVectorTokenArray: SuccinctBitVector? = null + private var englishReadingSuccinctBitVectorTangoLBS: SuccinctBitVector? = null + fun updateSystemUserDictionary( yomiTrie: LOUDSWithTermId?, tangoTrie: LOUDS?, @@ -195,6 +204,25 @@ class GraphBuilder { systemUserSuccinctBitVectorTangoLBS = succinctBitVectorTangoLBS } + fun updateEnglishReadingDictionary( + yomiTrie: LOUDSWithTermId?, + tangoTrie: LOUDS?, + tokenArray: TokenArray?, + succinctBitVectorLBSYomi: SuccinctBitVector?, + succinctBitVectorIsLeafYomi: SuccinctBitVector?, + succinctBitVectorTokenArray: SuccinctBitVector?, + succinctBitVectorTangoLBS: SuccinctBitVector?, + ) { + cachedGraph = null + englishReadingYomiTrie = yomiTrie + englishReadingTangoTrie = tangoTrie + englishReadingTokenArray = tokenArray + englishReadingSuccinctBitVectorLBSYomi = succinctBitVectorLBSYomi + englishReadingSuccinctBitVectorIsLeafYomi = succinctBitVectorIsLeafYomi + englishReadingSuccinctBitVectorTokenArray = succinctBitVectorTokenArray + englishReadingSuccinctBitVectorTangoLBS = succinctBitVectorTangoLBS + } + /** * グラフにノードを追加または更新する。 * 同じ終了位置に【同じ単語】かつ【同じ品詞ID(l/r)】のノードが既に存在する場合、 @@ -332,6 +360,7 @@ class GraphBuilder { val signature = conversionSignature( yomiTrie = yomiTrie, + englishReadingYomiTrie = englishReadingYomiTrie, wikiYomiTrie = wikiYomiTrie, webYomiTrie = webYomiTrie, personYomiTrie = personYomiTrie, @@ -371,6 +400,7 @@ class GraphBuilder { val systemTypoProgress = LinkedHashMap() val systemUserTypoProgress = LinkedHashMap() val systemPrefixStates = LinkedHashMap() + val englishReadingPrefixStates = LinkedHashMap() val systemUserPrefixStates = LinkedHashMap() val wikiPrefixStates = LinkedHashMap() val webPrefixStates = LinkedHashMap() @@ -884,6 +914,79 @@ class GraphBuilder { } } + // 3.1 英語読み辞書 + val localEnglishReadingYomiTrie = englishReadingYomiTrie + val localEnglishReadingTangoTrie = englishReadingTangoTrie + val localEnglishReadingTokenArray = englishReadingTokenArray + val localEnglishReadingLBSYomi = englishReadingSuccinctBitVectorLBSYomi + val localEnglishReadingIsLeafYomi = englishReadingSuccinctBitVectorIsLeafYomi + val localEnglishReadingTokenBitVector = englishReadingSuccinctBitVectorTokenArray + val localEnglishReadingTangoLBS = englishReadingSuccinctBitVectorTangoLBS + if ( + localEnglishReadingYomiTrie != null && + localEnglishReadingTangoTrie != null && + localEnglishReadingTokenArray != null && + localEnglishReadingLBSYomi != null && + localEnglishReadingIsLeafYomi != null && + localEnglishReadingTokenBitVector != null && + localEnglishReadingTangoLBS != null + ) { + val commonPrefixSearchEnglishReading = prefixSearch( + trie = localEnglishReadingYomiTrie, + bitVector = localEnglishReadingLBSYomi, + start = i, + previousStates = reusable?.englishReadingPrefixStates, + currentStates = englishReadingPrefixStates, + ) + if (commonPrefixSearchEnglishReading.isNotEmpty()) foundInAnyDictionary = true + for (prefixResult in commonPrefixSearchEnglishReading) { + val yomiStr = prefixResult.yomi + val nodeIndex = prefixResult.nodeIndex + if (nodeIndex <= 0) continue + val termId = localEnglishReadingYomiTrie.getTermId( + nodeIndex, + localEnglishReadingIsLeafYomi, + ) + val endIndex = i + yomiStr.length + if (endIndex <= reusablePrefixLength) continue + localEnglishReadingTokenArray.forEachDictionaryByYomiTermId( + termId, + localEnglishReadingTokenBitVector, + ) { posTableIndex, wordCost, tokenNodeId -> + val tango = when (tokenNodeId) { + -2 -> yomiStr + -1 -> yomiStr.hiraToKata() + else -> localEnglishReadingTangoTrie.getLetter( + tokenNodeId, + succinctBitVector = localEnglishReadingTangoLBS, + ) + } + val leftId = localEnglishReadingTokenArray.leftIds[posTableIndex.toInt()] + val cost = wordCost.toInt() + addOrUpdateNode( + graph, + endIndex, + Node( + l = leftId, + r = localEnglishReadingTokenArray.rightIds[posTableIndex.toInt()], + score = cost, + f = cost, + g = cost, + tango = tango, + yomiUsed = yomiStr, + len = yomiStr.length.toShort(), + sPos = i, + mozcAttributes = mozcAttributesFor(leftId), + ), + graphNodeDedupMode, + graphNodeTrace, + str, + "ENGLISH_READING", + ) + } + } + } + // 3.x システム辞書 (Typo Correction Prefix) if (enableTypoCorrectionJapaneseFlick && subStr().length > 2) { val typoProgress = typoSearch( @@ -1285,6 +1388,7 @@ class GraphBuilder { systemTypoProgress = systemTypoProgress, systemUserTypoProgress = systemUserTypoProgress, systemPrefixStates = systemPrefixStates, + englishReadingPrefixStates = englishReadingPrefixStates, systemUserPrefixStates = systemUserPrefixStates, wikiPrefixStates = wikiPrefixStates, webPrefixStates = webPrefixStates, @@ -1352,6 +1456,7 @@ class GraphBuilder { private fun conversionSignature( yomiTrie: LOUDSWithTermId, + englishReadingYomiTrie: LOUDSWithTermId?, wikiYomiTrie: LOUDSWithTermId?, webYomiTrie: LOUDSWithTermId?, personYomiTrie: LOUDSWithTermId?, @@ -1367,6 +1472,7 @@ class GraphBuilder { mozcNodeAttributeTable: MozcNodeAttributeTable?, ): Int { var result = System.identityHashCode(yomiTrie) + result = 31 * result + System.identityHashCode(englishReadingYomiTrie) result = 31 * result + System.identityHashCode(wikiYomiTrie) result = 31 * result + System.identityHashCode(webYomiTrie) result = 31 * result + System.identityHashCode(personYomiTrie) diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryModels.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryModels.kt index e1b5710e0..de2432253 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryModels.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryModels.kt @@ -13,6 +13,7 @@ enum class DictionaryCategory { READING_CORRECTION, KOTOWAZA, ENGLISH, + ENGLISH_READING, PERSON_NAME, PLACES, WIKI, @@ -75,6 +76,9 @@ enum class DictionaryFileKey { ENGLISH_WORD, ENGLISH_TOKEN, ENGLISH_QWERTY_GLIDE_INDEX, + ENGLISH_READING_TANGO, + ENGLISH_READING_YOMI, + ENGLISH_READING_TOKEN, PERSON_NAME_TANGO, PERSON_NAME_YOMI, PERSON_NAME_TOKEN, @@ -191,6 +195,10 @@ object DictionaryFileSpecs { triple(DictionaryFileKey.ENGLISH_TOKEN, DictionaryCategory.ENGLISH, DictionaryFileRole.ENGLISH_TOKEN, "english/token.dat.zip", DictionaryContentType.ENGLISH_TOKEN, R.string.external_dictionary_file_english_token, true), spec(DictionaryFileKey.ENGLISH_QWERTY_GLIDE_INDEX, DictionaryCategory.ENGLISH, DictionaryFileRole.ENGLISH_QWERTY_GLIDE_INDEX, "english/qwerty_glide_index.dat", DictionaryContentType.ENGLISH_QWERTY_GLIDE_INDEX, R.string.external_dictionary_file_english_word, false, false), + triple(DictionaryFileKey.ENGLISH_READING_TANGO, DictionaryCategory.ENGLISH_READING, DictionaryFileRole.TANGO, "english_reading/tango.dat.zip", DictionaryContentType.LOUDS, R.string.external_dictionary_file_tango, true), + triple(DictionaryFileKey.ENGLISH_READING_YOMI, DictionaryCategory.ENGLISH_READING, DictionaryFileRole.YOMI, "english_reading/yomi.dat.zip", DictionaryContentType.LOUDS_WITH_TERM_ID, R.string.external_dictionary_file_yomi, true), + triple(DictionaryFileKey.ENGLISH_READING_TOKEN, DictionaryCategory.ENGLISH_READING, DictionaryFileRole.TOKEN, "english_reading/token.dat.zip", DictionaryContentType.TOKEN_ARRAY, R.string.external_dictionary_file_token, true), + triple(DictionaryFileKey.PERSON_NAME_TANGO, DictionaryCategory.PERSON_NAME, DictionaryFileRole.TANGO, "person_name/tango_person_names.dat", DictionaryContentType.LOUDS, R.string.external_dictionary_file_tango, false), triple(DictionaryFileKey.PERSON_NAME_YOMI, DictionaryCategory.PERSON_NAME, DictionaryFileRole.YOMI, "person_name/yomi_person_names.dat", DictionaryContentType.LOUDS_WITH_TERM_ID, R.string.external_dictionary_file_yomi, false), triple(DictionaryFileKey.PERSON_NAME_TOKEN, DictionaryCategory.PERSON_NAME, DictionaryFileRole.TOKEN, "person_name/token_person_names.dat", DictionaryContentType.TOKEN_ARRAY, R.string.external_dictionary_file_token, false), @@ -266,6 +274,7 @@ fun DictionaryCategory.isDisableableBundledDictionary(): Boolean = this in setOf( DictionaryCategory.READING_CORRECTION, DictionaryCategory.KOTOWAZA, + DictionaryCategory.ENGLISH_READING, DictionaryCategory.PERSON_NAME, DictionaryCategory.PLACES, DictionaryCategory.WIKI, diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideStore.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideStore.kt index 239696848..219a02967 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideStore.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideStore.kt @@ -168,7 +168,9 @@ class DictionaryOverrideStore private constructor( fun removeAllOverrides() { val fileChanged = directory.exists() && directory.deleteRecursively() val stateChanged = prefs.all.any { (key, value) -> isResetAllStateChange(key, value) } - if (fileChanged || stateChanged) { + val englishReadingPreferenceChanged = + defaultPrefs.contains(ENGLISH_READING_ENABLED_PREFERENCE) + if (fileChanged || stateChanged || englishReadingPreferenceChanged) { synchronized(this) { val nextRevision = currentRevision + 1L prefs.edit() @@ -176,6 +178,9 @@ class DictionaryOverrideStore private constructor( .putLong(REVISION_PREF_KEY, nextRevision) .apply() } + if (englishReadingPreferenceChanged) { + defaultPrefs.edit().remove(ENGLISH_READING_ENABLED_PREFERENCE).apply() + } } OptionalDictionaryMigration(defaultPrefs, prefs).migrateIfNeeded(force = true) } @@ -216,13 +221,22 @@ class DictionaryOverrideStore private constructor( prefs.getBoolean(keyExternalEnabledKey(key), false) fun isOptionalBundledEnabled(category: DictionaryCategory): Boolean = - prefs.getBoolean( - optionalBundledEnabledKey(category), - category in setOf(DictionaryCategory.READING_CORRECTION, DictionaryCategory.KOTOWAZA), - ) + if (category == DictionaryCategory.ENGLISH_READING) { + defaultPrefs.getBoolean(ENGLISH_READING_ENABLED_PREFERENCE, true) + } else { + prefs.getBoolean( + optionalBundledEnabledKey(category), + category in setOf(DictionaryCategory.READING_CORRECTION, DictionaryCategory.KOTOWAZA), + ) + } fun setOptionalBundledEnabled(category: DictionaryCategory, enabled: Boolean) { if (isOptionalBundledEnabled(category) == enabled) return + if (category == DictionaryCategory.ENGLISH_READING) { + defaultPrefs.edit().putBoolean(ENGLISH_READING_ENABLED_PREFERENCE, enabled).apply() + applyRevisionedEdit { true } + return + } applyRevisionedEdit { putBoolean(optionalBundledEnabledKey(category), enabled) true @@ -281,6 +295,7 @@ class DictionaryOverrideStore private constructor( when (category) { DictionaryCategory.READING_CORRECTION, DictionaryCategory.KOTOWAZA -> true + DictionaryCategory.ENGLISH_READING -> true DictionaryCategory.PERSON_NAME -> defaultPrefs.getBoolean("mozc_ut_person_name_preference", false) DictionaryCategory.PLACES -> @@ -318,6 +333,8 @@ class DictionaryOverrideStore private constructor( private const val PREF_NAME = "dictionary_override_store" private const val DIRECTORY_NAME = "dictionary_overrides" const val REVISION_PREF_KEY = "dictionary_override_revision" + const val ENGLISH_READING_ENABLED_PREFERENCE = + "english_reading_dictionary_enable_preference" private val metadataType = object : TypeToken() {}.type fun metadataPrefKey(key: DictionaryFileKey) = "metadata_${key.name}" diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt index aeadc8bcd..53e41779a 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt @@ -82,10 +82,15 @@ class DictionarySourceResolver @Inject constructor( (store.getOverrideMetadata(it.key)?.validationStatus == ValidationStatus.INVALID) } - if (hasInvalid) return DictionaryCategoryLoadState.Invalid if (specs.all { !it.partOfTripleDictionary }) { return DictionaryCategoryLoadState.Bundled } + if (category == DictionaryCategory.ENGLISH_READING && + !store.isOptionalBundledEnabled(category) + ) { + return DictionaryCategoryLoadState.Disabled + } + if (hasInvalid) return DictionaryCategoryLoadState.Invalid if (hasAny && (!hasAll || !validAll)) return DictionaryCategoryLoadState.Partial if (validAll && store.isExternalEnabledForCategory(category)) return DictionaryCategoryLoadState.User if (category.isDisableableBundledDictionary()) { diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryZipEntryNameMapper.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryZipEntryNameMapper.kt index a7dd1539b..4875015ab 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryZipEntryNameMapper.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryZipEntryNameMapper.kt @@ -40,6 +40,12 @@ data class DictionaryZipMappedEntry( ) object DictionaryZipEntryNameMapper { + private val pathSuffixToKey: Map = mapOf( + "english_reading/tango.dat.zip" to DictionaryFileKey.ENGLISH_READING_TANGO, + "english_reading/yomi.dat.zip" to DictionaryFileKey.ENGLISH_READING_YOMI, + "english_reading/token.dat.zip" to DictionaryFileKey.ENGLISH_READING_TOKEN, + ) + private val filenameToKey: Map = mapOf( "tango.dat" to DictionaryFileKey.SYSTEM_TANGO, "yomi.dat" to DictionaryFileKey.SYSTEM_YOMI, @@ -104,7 +110,13 @@ object DictionaryZipEntryNameMapper { return DictionaryZipEntryNameResolution.Ignored(entryName, "Ignored macOS metadata entry") } - val key = filenameToKey[basename.lowercase(Locale.ROOT)] + val normalizedPath = pathSegments.joinToString("/").lowercase(Locale.ROOT) + val key = pathSuffixToKey.entries + .firstOrNull { (suffix, _) -> + normalizedPath == suffix || normalizedPath.endsWith("/$suffix") + } + ?.value + ?: filenameToKey[basename.lowercase(Locale.ROOT)] ?: return DictionaryZipEntryNameResolution.Unrecognized(entryName, basename) return DictionaryZipEntryNameResolution.Mapped(key, entryName, basename) } diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/ime_service/IMEService.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/ime_service/IMEService.kt index 35c47950d..0de4db03f 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/ime_service/IMEService.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/ime_service/IMEService.kt @@ -202,6 +202,7 @@ import com.kazumaproject.markdownhelperkeyboard.databinding.FloatingKeyboardLayo import com.kazumaproject.markdownhelperkeyboard.databinding.MainLayoutBinding import com.kazumaproject.markdownhelperkeyboard.dictionary_override.DictionaryBinaryReader import com.kazumaproject.markdownhelperkeyboard.dictionary_override.DictionaryCategory +import com.kazumaproject.markdownhelperkeyboard.dictionary_override.DictionaryCategoryLoadState import com.kazumaproject.markdownhelperkeyboard.dictionary_override.DictionaryOverrideStore import com.kazumaproject.markdownhelperkeyboard.dictionary_override.DictionarySourceResolver import com.kazumaproject.markdownhelperkeyboard.gemma.GemmaImageCapability @@ -336,8 +337,10 @@ import com.kazumaproject.tenkey.extensions.isHiragana import com.kazumaproject.tenkey.extensions.isLatinAlphabet import com.kazumaproject.tenkey.extensions.toggleDakutenWithSeion import com.kazumaproject.tenkey.extensions.toggleHandakutenWithSeion +import dagger.Lazy import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview @@ -494,7 +497,10 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, lateinit var inputMethodManager: InputMethodManager @Inject - lateinit var kanaKanjiEngine: KanaKanjiEngine + lateinit var kanaKanjiEngineProvider: Lazy + + private lateinit var kanaKanjiEngine: KanaKanjiEngine + private val kanaKanjiEngineReady = CompletableDeferred() @Inject lateinit var ngramRuleScorerManager: NgramRuleScorerManager @@ -761,6 +767,8 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, private val selectedEditorTextRequestId = AtomicLong(0L) private val editorConnectionReadMutex = Mutex() private var systemUserDictionaryLoadJob: Job? = null + private var kanaKanjiEngineLoadJob: Job? = null + private var kanaKanjiEngineActivationJob: Job? = null private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob()) private val ioScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) @@ -1340,6 +1348,9 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, @Volatile private var lastAppliedDictionaryOverrideRevision: Long = Long.MIN_VALUE + @Volatile + private var lastAppliedEnglishReadingEnabled: Boolean? = null + @Volatile private var dictionaryOverrideApplyJob: Job? = null @@ -1698,6 +1709,7 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, private var candidateTabVisibility: Boolean? = false private var conversionBackend: ConversionBackend = ConversionBackend.LEGACY private var predictionConfig: PredictionConfig = PredictionConfig() + @Volatile private var kanaKanjiConversionSession: KanaKanjiConversionSession? = null private val candidateRequestTracker = CandidateRequestTracker() private var symbolKeyboardFirstItem: SymbolMode? = SymbolMode.EMOJI @@ -2138,6 +2150,55 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, val symbolKeyboard: CustomSymbolKeyboardView? ) + private fun startKanaKanjiEngineLoad() { + if (kanaKanjiEngineReady.isCompleted || kanaKanjiEngineLoadJob?.isActive == true) return + kanaKanjiEngineLoadJob = ioScope.launch { + val startedAt = System.nanoTime() + try { + val engine = kanaKanjiEngineProvider.get() + withContext(Dispatchers.Main.immediate) { + kanaKanjiEngine = engine + } + kanaKanjiEngineReady.complete(engine) + Timber.d( + "KanaKanjiEngine core dictionary load complete: " + + "elapsed_ms=${(System.nanoTime() - startedAt) / 1_000_000.0} " + + "thread=${Thread.currentThread().name}" + ) + } catch (cancellation: CancellationException) { + throw cancellation + } catch (throwable: Throwable) { + Timber.e(throwable, "Failed to load core dictionaries asynchronously") + kanaKanjiEngineReady.completeExceptionally(throwable) + } + } + } + + private suspend fun awaitKanaKanjiEngineOrNull(): KanaKanjiEngine? = try { + kanaKanjiEngineReady.await() + } catch (cancellation: CancellationException) { + throw cancellation + } catch (throwable: Throwable) { + Timber.e(throwable, "KanaKanjiEngine is unavailable") + null + } + + private fun activateKanaKanjiEngineWhenReady( + preferences: ImePreferencesSnapshot, + inputSessionId: Long, + ) { + kanaKanjiEngineActivationJob?.cancel() + kanaKanjiEngineActivationJob = scope.launch { + if (awaitKanaKanjiEngineOrNull() == null) return@launch + if (inputSessionId != flickPreviewEditorSessionId) return@launch + if (kanaKanjiConversionSession == null) { + startKanaKanjiConversionSession(preferences.conversionBackend) + } + initializeMozcDictionaries(preferences) + refreshCandidateStripContent() + } + } + override fun onCreate() { super.onCreate() Timber.d("onCreate") @@ -2150,6 +2211,7 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, ) runtimeInputPreferenceListenerRegistered = true syncRuntimeInputPreferences() + startKanaKanjiEngineLoad() if (AppVariantConfig.hasGemma) { scope.launch { @@ -2401,9 +2463,15 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, ) ) applyImePreferences(preferences) - startKanaKanjiConversionSession(preferences.conversionBackend) + conversionBackend = preferences.conversionBackend + kanaKanjiConversionSession = null + candidateRequestTracker.restart(preferences.conversionBackend) + candidateRefreshCoordinator.restart() + activateKanaKanjiEngineWhenReady( + preferences = preferences, + inputSessionId = flickPreviewEditorSessionId, + ) resetKeyboard() - initializeMozcDictionaries(preferences) refreshClipboardPreviewSnapshot() syncCustomKeyboardSuggestionPreference() refreshCandidateStripContent() @@ -2940,11 +3008,22 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, private fun applyDictionaryOverrideRevisionIfNeeded() { val currentRevision = dictionaryOverrideStore.currentRevision - if (currentRevision == lastAppliedDictionaryOverrideRevision) return + val englishReadingEnabled = + dictionarySourceResolver.resolveCategoryLoadState(DictionaryCategory.ENGLISH_READING) != + DictionaryCategoryLoadState.Disabled + if (currentRevision == lastAppliedDictionaryOverrideRevision && + englishReadingEnabled == lastAppliedEnglishReadingEnabled + ) return if (dictionaryOverrideApplyJob?.isActive == true) return + // Asset opening and dictionary decoding must stay on IO. Only the lightweight candidate + // refresh at the end of this job is dispatched back to the main thread. dictionaryOverrideApplyJob = ioScope.launch { val revisionToApply = dictionaryOverrideStore.currentRevision + val englishReadingEnabledToApply = + dictionarySourceResolver.resolveCategoryLoadState( + DictionaryCategory.ENGLISH_READING + ) != DictionaryCategoryLoadState.Disabled val initialOptionalStateOnly = lastAppliedDictionaryOverrideRevision == Long.MIN_VALUE && !kanaKanjiEngine.isInitialOptionalDictionaryStateLoaded() @@ -2970,8 +3049,17 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, Timber.w(it, "Failed to apply dictionary override revision $revisionToApply") }.isSuccess - if (success && dictionaryOverrideStore.currentRevision == revisionToApply) { + if ( + success && + dictionaryOverrideStore.currentRevision == revisionToApply && + ( + dictionarySourceResolver.resolveCategoryLoadState( + DictionaryCategory.ENGLISH_READING + ) != DictionaryCategoryLoadState.Disabled + ) == englishReadingEnabledToApply + ) { lastAppliedDictionaryOverrideRevision = revisionToApply + lastAppliedEnglishReadingEnabled = englishReadingEnabledToApply if (initialOptionalStateOnly) { Timber.d( "Initial optional dictionary state loaded without core reload: " + @@ -2994,6 +3082,14 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, ) } } + + if (success) { + withContext(Dispatchers.Main.immediate) { + if (isInputViewActive) { + requestCandidateRefresh(CandidateShowFlag.Updating) + } + } + } } } @@ -4782,6 +4878,7 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, qwertyGlideInputCoordinator?.cancelPending() englishEngine.cancelQwertyGlideWarmup() clearZenzLiveSlot("onDestroy") + kanaKanjiEngineActivationJob?.cancel() suggestionAdapter?.release() suggestionAdapter = null shortcutAdapter = null @@ -4798,12 +4895,16 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, hasHardwareKeyboardConnected = null clipboardManager.removePrimaryClipChangedListener(clipboardListener) filteredCandidateList = null - if (mozcUTPersonName == true) kanaKanjiEngine.releasePersonNamesDictionary() - if (mozcUTPlaces == true) kanaKanjiEngine.releasePlacesDictionary() - if (mozcUTWiki == true) kanaKanjiEngine.releaseWikiDictionary() - if (mozcUTNeologd == true) kanaKanjiEngine.releaseNeologdDictionary() - if (mozcUTWeb == true) kanaKanjiEngine.releaseWebDictionary() - if (kanaKanjiEngine.isSystemUserDictionaryInitialized()) kanaKanjiEngine.releaseSystemUserDictionary() + if (::kanaKanjiEngine.isInitialized) { + if (mozcUTPersonName == true) kanaKanjiEngine.releasePersonNamesDictionary() + if (mozcUTPlaces == true) kanaKanjiEngine.releasePlacesDictionary() + if (mozcUTWiki == true) kanaKanjiEngine.releaseWikiDictionary() + if (mozcUTNeologd == true) kanaKanjiEngine.releaseNeologdDictionary() + if (mozcUTWeb == true) kanaKanjiEngine.releaseWebDictionary() + if (kanaKanjiEngine.isSystemUserDictionaryInitialized()) { + kanaKanjiEngine.releaseSystemUserDictionary() + } + } isFlickOnlyMode = null isOmissionSearchEnable = null delayTime = null @@ -20494,14 +20595,15 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, } private suspend fun setSymbols(mainView: MainLayoutBinding) { + val engine = awaitKanaKanjiEngineOrNull() ?: return coroutineScope { if (cachedEmoji == null || cachedEmoticons == null || cachedSymbols == null) { val emojiDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolEmojiCandidates() } + async(Dispatchers.Default) { engine.getSymbolEmojiCandidates() } val emoticonDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolEmoticonCandidates() } + async(Dispatchers.Default) { engine.getSymbolEmoticonCandidates() } val symbolDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolCandidates() } + async(Dispatchers.Default) { engine.getSymbolCandidates() } cachedEmoji = emojiDeferred.await() cachedEmoticons = emoticonDeferred.await() cachedSymbols = symbolDeferred.await() @@ -20524,14 +20626,15 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, } private suspend fun setSymbolsClipboard(mainView: MainLayoutBinding) { + val engine = awaitKanaKanjiEngineOrNull() ?: return coroutineScope { if (cachedEmoji == null || cachedEmoticons == null || cachedSymbols == null) { val emojiDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolEmojiCandidates() } + async(Dispatchers.Default) { engine.getSymbolEmojiCandidates() } val emoticonDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolEmoticonCandidates() } + async(Dispatchers.Default) { engine.getSymbolEmoticonCandidates() } val symbolDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolCandidates() } + async(Dispatchers.Default) { engine.getSymbolCandidates() } cachedEmoji = emojiDeferred.await() cachedEmoticons = emoticonDeferred.await() cachedSymbols = symbolDeferred.await() @@ -20554,14 +20657,15 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, } private suspend fun setSymbolsFloating(floatingKeyboardLayoutBinding: FloatingKeyboardLayoutBinding) { + val engine = awaitKanaKanjiEngineOrNull() ?: return coroutineScope { if (cachedEmoji == null || cachedEmoticons == null || cachedSymbols == null) { val emojiDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolEmojiCandidates() } + async(Dispatchers.Default) { engine.getSymbolEmojiCandidates() } val emoticonDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolEmoticonCandidates() } + async(Dispatchers.Default) { engine.getSymbolEmoticonCandidates() } val symbolDeferred = - async(Dispatchers.Default) { kanaKanjiEngine.getSymbolCandidates() } + async(Dispatchers.Default) { engine.getSymbolCandidates() } cachedEmoji = emojiDeferred.await() cachedEmoticons = emoticonDeferred.await() cachedSymbols = symbolDeferred.await() @@ -23024,9 +23128,11 @@ class IMEService : InputMethodService(), LifecycleOwner, InputConnection, typoCorrectionJapaneseFlickEnabled: Boolean = false, typoCorrectionQwertyEnglishEnabled: Boolean = false, ): KanaKanjiQueryResult { + val engine = awaitKanaKanjiEngineOrNull() + ?: return KanaKanjiQueryResult(candidates = emptyList()) awaitSystemUserDictionaryLoad() val session = kanaKanjiConversionSession ?: KanaKanjiConversionSession( - engine = kanaKanjiEngine, + engine = engine, backend = conversionBackend, ).also { kanaKanjiConversionSession = it diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/setting_activity/ui/external_dictionary/ExternalDictionarySettingsFragment.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/setting_activity/ui/external_dictionary/ExternalDictionarySettingsFragment.kt index 8ac2554ed..1fe7aaa1b 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/setting_activity/ui/external_dictionary/ExternalDictionarySettingsFragment.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/setting_activity/ui/external_dictionary/ExternalDictionarySettingsFragment.kt @@ -597,6 +597,7 @@ class ExternalDictionarySettingsFragment : PreferenceFragmentCompat() { DictionaryCategory.READING_CORRECTION -> "reading_correction" DictionaryCategory.KOTOWAZA -> "kotowaza" DictionaryCategory.ENGLISH -> getString(R.string.external_dictionary_category_english) + DictionaryCategory.ENGLISH_READING -> getString(R.string.external_dictionary_category_english_reading) DictionaryCategory.PERSON_NAME -> getString(R.string.mozc_ut_person_name_title) DictionaryCategory.PLACES -> getString(R.string.mozc_ut_places_title) DictionaryCategory.WIKI -> getString(R.string.mozc_ut_wiki_title) @@ -637,6 +638,7 @@ private val DICTIONARY_CATEGORIES = listOf( DictionaryCategory.EMOTICON, DictionaryCategory.SYMBOL, DictionaryCategory.ENGLISH, + DictionaryCategory.ENGLISH_READING, DictionaryCategory.READING_CORRECTION, DictionaryCategory.KOTOWAZA, DictionaryCategory.PERSON_NAME, diff --git a/app/src/main/res/values-ja/strings.xml b/app/src/main/res/values-ja/strings.xml index 5aa512e19..d4e7d92e0 100644 --- a/app/src/main/res/values-ja/strings.xml +++ b/app/src/main/res/values-ja/strings.xml @@ -1055,6 +1055,7 @@ 顔文字辞書 記号辞書 英語辞書 + 英語読み辞書 外部ファイルで置き換える 内蔵辞書を使用 この辞書を使用する @@ -1542,6 +1543,8 @@ 入力した読みより長い日本語候補を表示します。 英語の予測変換 英語辞書の前方一致候補を表示します。OFFでも入力文字と大文字・小文字候補は残ります。 + 英語読み辞書 + 「かー」から「car」のような英語候補を表示します。 システム辞書の予測変換 内蔵システム辞書を日本語の前方一致予測に使用します。 システムユーザー辞書 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 477de11f6..814fdaa95 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -976,6 +976,7 @@ Emoticon dictionary Symbol dictionary English dictionary + English reading dictionary Replace with external file Use bundled dictionary Use this dictionary @@ -1582,6 +1583,8 @@ Show Japanese candidates whose reading continues beyond the entered text. English prediction Show English dictionary completions. Entered text and capitalization variants remain available when off. + English reading dictionary + Use English candidates such as “car” for kana readings such as “かー”. System dictionary prediction Use the built-in system dictionary for Japanese completion candidates. System user dictionary diff --git a/app/src/main/res/xml/pref_conversion_engine.xml b/app/src/main/res/xml/pref_conversion_engine.xml index 6fa177d68..c0e9a4f03 100644 --- a/app/src/main/res/xml/pref_conversion_engine.xml +++ b/app/src/main/res/xml/pref_conversion_engine.xml @@ -30,6 +30,13 @@ android:title="@string/english_prediction_enable_title" app:iconSpaceReserved="false" /> + + diff --git a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideCoreTest.kt b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideCoreTest.kt index 7068ed54f..0c83fe762 100644 --- a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideCoreTest.kt +++ b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryOverrideCoreTest.kt @@ -227,6 +227,26 @@ class DictionaryOverrideCoreTest { assertEquals(before + 2L, store.currentRevision) } + @Test + fun englishReadingDictionary_isEnabledByDefaultAndCanBeDisabled() { + val store = alwaysValidStore("english-reading-setting") + val resolver = sourceResolver(store) + + assertTrue(store.isOptionalBundledEnabled(DictionaryCategory.ENGLISH_READING)) + assertEquals( + DictionaryCategoryLoadState.Bundled, + resolver.resolveCategoryLoadState(DictionaryCategory.ENGLISH_READING), + ) + + store.setOptionalBundledEnabled(DictionaryCategory.ENGLISH_READING, false) + + assertFalse(store.isOptionalBundledEnabled(DictionaryCategory.ENGLISH_READING)) + assertEquals( + DictionaryCategoryLoadState.Disabled, + resolver.resolveCategoryLoadState(DictionaryCategory.ENGLISH_READING), + ) + } + @Test fun revision_tripleDictionaryFinalSaveAutoEnableUsesSingleIncrementForThatSave() { val store = alwaysValidStore("revision-triple") @@ -771,6 +791,18 @@ class DictionaryOverrideCoreTest { assertEquals(DictionaryFileKey.CONNECTION_ID, DictionaryZipEntryNameMapper.map("connectionId.dat")) assertEquals(DictionaryFileKey.CONNECTION_ID, DictionaryZipEntryNameMapper.map("connectionid.dat")) assertEquals(DictionaryFileKey.CONNECTION_ID, DictionaryZipEntryNameMapper.map("connectionId.dat.zip")) + assertEquals( + DictionaryFileKey.ENGLISH_READING_TANGO, + DictionaryZipEntryNameMapper.map("app/src/main/assets/english_reading/tango.dat.zip"), + ) + assertEquals( + DictionaryFileKey.ENGLISH_READING_YOMI, + DictionaryZipEntryNameMapper.map("english_reading\\yomi.dat.zip"), + ) + assertEquals( + DictionaryFileKey.ENGLISH_READING_TOKEN, + DictionaryZipEntryNameMapper.map("english_reading/token.dat.zip"), + ) assertEquals(DictionaryFileKey.POS_TABLE, DictionaryZipEntryNameMapper.map("pos_table.dat")) assertEquals(DictionaryFileKey.ID_DEF, DictionaryZipEntryNameMapper.map("id.def")) assertNull(DictionaryZipEntryNameMapper.map("unknown.dat"))