Skip to content

Commit

Permalink
remove duplicate translations and show # missing
Browse files Browse the repository at this point in the history
  • Loading branch information
@jillesvangurp
jillesvangurp committed Jun 9, 2024
1 parent 5773a60 commit 1173fbf
Show file tree
Hide file tree
Showing 4 changed files with 158 additions and 88 deletions.
23 changes: 23 additions & 0 deletions src/commonMain/kotlin/com/jillesvangurp/fluentai/FluentFile.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,3 +151,26 @@ fun String.parseFluent(): List<FluentChunk> {
}
}
}

fun List<FluentFile>.missingTranslations(preferred: FluentFile): Map<String, Int> {
val others = filter { it.name != preferred.name }
return preferred.keys().map { key ->
val source = preferred[key]?: error("key should have definition")
key to others.mapNotNull {
it[key]?.takeIf { it.definition != source.definition }
}.size.let { others.size - it }
}.toMap()
}

fun List<FluentFile>.master(masterLanguage: String) = firstOrNull { it.matches(masterLanguage) }

fun List<FluentFile>.cleanupTranslations(masterLanguage: String): List<FluentFile> {
return master(masterLanguage)?.let { master ->
(filter { it.name != master.name }.map { file ->
val cleaned = file.asMap().entries.filter {(id,chunk) ->
master[id]?.definition != chunk.definition && chunk.definition.isNotBlank()
}.map { it.value }
FluentFile(file.name, cleaned.associateBy { it.id }.sortedContent())
} + master).sortedBy { it.name }
}?: this
}
24 changes: 16 additions & 8 deletions src/jsMain/kotlin/files/filemanagement.kt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package files

import com.jillesvangurp.fluentai.FluentFile
import com.jillesvangurp.fluentai.cleanupTranslations
import com.jillesvangurp.fluentai.sortedContent
import components.confirm
import components.downloadButton
Expand Down Expand Up @@ -31,6 +32,7 @@ import org.w3c.dom.HTMLTextAreaElement
import org.w3c.files.FileReader
import org.w3c.files.get
import settings.SettingsStore
import settings.preferredTranslationLanguage
import utils.handlerScope
import withKoin

Expand Down Expand Up @@ -93,18 +95,22 @@ fun RenderContext.fileManager() {
}

suspend fun FluentFilesStore.loadOwnFtls() {
val files = Locales.entries.mapNotNull { locale ->
TranslationStore.fetchFtl(locale.id)?.let {
FluentFile(locale.id, it)
}
}.sortedBy { it.name }
persistAndUpdate(files)
withKoin {
val settingsStore = get<SettingsStore>()
val files = Locales.entries.mapNotNull { locale ->
TranslationStore.fetchFtl(locale.id)?.let {
FluentFile(locale.id, it)
}
}.sortedBy { it.name }
persistAndUpdate(files.cleanupTranslations(settingsStore.current.preferredTranslationLanguage))
}
}

fun RenderContext.fileLoader() {
withKoin {
// Drag target store
val fileContentStore = get<FluentFilesStore>()
val settingsStore = get<SettingsStore>()

// Drag target
div(
Expand Down Expand Up @@ -149,11 +155,13 @@ fun RenderContext.fileLoader() {
while (processedFiles > 0) {
delay(5.milliseconds)
}
val pl = settingsStore.current.preferredTranslationLanguage

fileContentStore.persistAndUpdate(
loadedFiles.map {
// sort on load
FluentFile(it.name, it.asMap().sortedContent())
},
}.cleanupTranslations(settingsStore.current.preferredTranslationLanguage),
)
}

Expand Down Expand Up @@ -263,7 +271,7 @@ fun RenderContext.fileStats(file: FluentFile) {
val fileStore = get<FluentFilesStore>()
fileStore.data.filterNotNull().render { files ->
settingsStore.data.render { settings ->
val preferredLanguage = settings?.translationSourceLanguage ?: "en-US"
val preferredLanguage = settings.preferredTranslationLanguage
val source = files.firstOrNull { it.matches(preferredLanguage) }
ul {
val numberOfTranslations = file.chunks.size
Expand Down
197 changes: 117 additions & 80 deletions src/jsMain/kotlin/fluenteditor/editor.kt
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package fluenteditor

import ai.TranslationService
import com.jillesvangurp.fluentai.FluentFile
import com.jillesvangurp.fluentai.groupIdsByLargestPrefix
import com.jillesvangurp.fluentai.missingTranslations
import components.confirm
import components.fadeInFadeoutTransition
import components.primaryButton
Expand All @@ -24,6 +26,7 @@ import localization.TL
import localization.getTranslationString
import localization.translate
import settings.SettingsStore
import settings.preferredTranslationLanguage
import withKoin

fun RenderContext.fluentBrowser() {
Expand All @@ -36,75 +39,90 @@ fun RenderContext.fluentBrowser() {
selectedTranslationEditor(selectedIdStore)
}
}

}

fun RenderContext.idsListComponent(selectedIdStore: Store<String>) {
withKoin {
val fluentFilesStore = get<FluentFilesStore>()
fluentFilesStore.data.filterNotNull().render { files ->
val settingsStore = get<SettingsStore>()
settingsStore.data.render {settings->
fluentFilesStore.data.filterNotNull().render { files ->
val missingTranslations =
files .firstOrNull { it.matches(settings.preferredTranslationLanguage) }
?.let { preferred ->
files.missingTranslations(preferred)
}.orEmpty()

val keys = files.flatMap { it.keys() }.distinct().sorted()
val keys = files.flatMap { it.keys() }.distinct().sorted()

div("w-96 flex flex-col gap-2 p-5 bg-white shadow-lg m-2") {
secondaryButton(text = TL.Common.Add, iconSource = SvgIconSource.Plus) {
clicks handledBy {
selectedIdStore.update("")
div("w-96 flex flex-col gap-2 p-5 bg-white shadow-lg m-2") {
secondaryButton(text = TL.Common.Add, iconSource = SvgIconSource.Plus) {
clicks handledBy {
selectedIdStore.update("")
}
}
}
val searchStore = storeOf("")
val searchStore = storeOf("")

twInputField(
searchStore,
null,
getTranslationString(TL.Common.FilterPlaceholder),
)
twInputField(
searchStore,
null,
getTranslationString(TL.Common.FilterPlaceholder),
)

div("grow overflow-y-auto") {
div("grow overflow-y-auto") {

div("max-h-0") {
searchStore.data.render { query ->
keys.filter { translationId ->
if (query.isBlank()) true
else {
translationId.lowercase().contains(query.lowercase())
}
}.also {
p {
+"Total ${it.size}"
}
}.groupIdsByLargestPrefix().forEach { (prefix, ids) ->
val showIdsStore = storeOf(true)
showIdsStore.data.render { show ->
div {
a {
+"${prefix.takeIf { it.isNotBlank() } ?: "-terms"} (${ids.size})"
div("max-h-0") {
searchStore.data.render { query ->
keys.filter { translationId ->
if (query.isBlank()) true
else {
translationId.lowercase().contains(query.lowercase())
}
}.also {
p {
+"Total ${it.size}"
}
}.groupIdsByLargestPrefix().forEach { (prefix, ids) ->
val showIdsStore = storeOf(true)
showIdsStore.data.render { show ->
div {
a {
+"${prefix.takeIf { it.isNotBlank() } ?: "-terms"} (${ids.size})"

clicks handledBy {
showIdsStore.update(!showIdsStore.current)
clicks handledBy {
showIdsStore.update(!showIdsStore.current)
}
}
}
}
if (show) {
ids.forEach { translationId ->
div("ml-5") {
a {
+translationId.replace(
"$prefix-".takeIf { it != "-" } ?: "",
"",
)
selectedIdStore.data.render {
if (selectedIdStore.current == translationId) {
+" *"
if (show) {
ids.forEach { translationId ->
div("ml-5") {
a {
+translationId.replace(
"$prefix-".takeIf { it != "-" } ?: "",
"",
)
selectedIdStore.data.render {
if (selectedIdStore.current == translationId) {
+" *"
}
}

clicks handledBy {
if (selectedIdStore.current == translationId) {
selectedIdStore.update("")
} else {
selectedIdStore.update(
translationId,
)
}
}
}
clicks handledBy {
if (selectedIdStore.current == translationId) {
selectedIdStore.update("")
} else {
selectedIdStore.update(
translationId,
)
missingTranslations[translationId]?.let {missing ->
if(missing>0) {
span {
+" missing $missing"
}
}
}
}
Expand All @@ -117,7 +135,6 @@ fun RenderContext.idsListComponent(selectedIdStore: Store<String>) {
}
}
}

}
}
}
Expand Down Expand Up @@ -191,8 +208,7 @@ fun RenderContext.selectedTranslationEditor(
) {
val originalText = files.first {
it.matches(
settingsStore.current?.translationSourceLanguage
?: "en-US",
settingsStore.current.preferredTranslationLanguage,
)
}[translationId]?.definition.orEmpty()
val isDisabled = !translationService.enabled()
Expand All @@ -204,27 +220,12 @@ fun RenderContext.selectedTranslationEditor(
}

clicks handledBy {
runWithBusy(
{
translationService.translate(
originalText,
file.name,
)
},
translationArgs = mapOf("language" to file.name),
) { translated ->
translationEditor.update(
translated ?: ":-(",
)
if (translated != null) {
val newFile =
file.put(translationId, translated)
fluentFilesStore.addOrReplace(newFile)
translationEditor.update(newFile[translationId]?.definition.orEmpty())
} else {
console.error("no translation")
}
}
translateWithOpenAI(
translationId,
originalText,
file,
translationEditor,
)
}
}
primaryButton(
Expand Down Expand Up @@ -252,6 +253,40 @@ fun RenderContext.selectedTranslationEditor(
}
}

private suspend fun translateWithOpenAI(
translationId: String,
originalText: String,
file: FluentFile,
translationEditor: Store<String>
) {
withKoin {
val fluentFilesStore = get<FluentFilesStore>()
val translationService = get<TranslationService>()

runWithBusy(
{
translationService.translate(
originalText,
file.name,
)
},
translationArgs = mapOf("language" to file.name),
) { translated ->
translationEditor.update(
translated ?: ":-(",
)
if (translated != null) {
val newFile =
file.put(translationId, translated)
fluentFilesStore.addOrReplace(newFile)
translationEditor.update(newFile[translationId]?.definition.orEmpty())
} else {
console.error("no translation")
}
}
}
}

private fun RenderContext.createNewTranslationId() {
withKoin {
val fluentFilesStore = get<FluentFilesStore>()
Expand Down Expand Up @@ -288,15 +323,17 @@ private fun RenderContext.createNewTranslationId() {
newIdStore.data.render { newId ->
newTranslationStore.data.render { newTranslation ->
disabled(
files.isNullOrEmpty() || newTranslation.isBlank() || newId.isBlank() || files.orEmpty()
.flatMap { it.keys() }.contains(newId),
files.isNullOrEmpty()
|| newTranslation.isBlank()
|| newId.isBlank()
|| files.flatMap { it.keys() }.contains(newId),
)
}
}

clicks handledBy {
val newId = newIdStore.current
val preferred = settings?.translationSourceLanguage ?: "en"
val preferred = settings.preferredTranslationLanguage
console.log(preferred)
val fluentFile =
files?.firstOrNull { it.matches(preferred) } ?: files?.first()
Expand Down
2 changes: 2 additions & 0 deletions src/jsMain/kotlin/settings/settings.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ data class Settings(
val openAIKey: String? = null,
)

val Settings?.preferredTranslationLanguage get() = this?.translationSourceLanguage?:"en-US"

class SettingsStore : LocalStoringStore<Settings>(Settings(), "settings", Settings.serializer()) {
fun setLocale(locale: Locales) {
persistAndUpdate(
Expand Down

0 comments on commit 1173fbf

Please sign in to comment.