Use normalization wrapper in js-invokable (#2068)

myieye · web-flow · commit 388d4c024bba · 2025-10-27T10:27:38.000+01:00
* Use unicode normalized api in tests by default

* Move normalization wrapper to correct namespace

* Use normalization wrapper in MiniLcmJsInvokable

* Extend query tests to cover normalization
diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/BasicApiTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/BasicApiTests.cs
@@ -22,7 +22,7 @@ public async Task GetEntry_WithNullLexemeFormOA_CreatesNewLexemeForm()
             LexemeForm = new MultiString { { "en", "test" } }
         });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id);
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set LexemeFormOA to null",
             "Restore LexemeFormOA",
diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/ComplexFormComponentTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/ComplexFormComponentTests.cs
@@ -27,7 +27,7 @@ protected override Task<IMiniLcmApi> NewApi()
     public override async Task InitializeAsync()
     {
         await base.InitializeAsync();
-        var fwDataApi = (FwDataMiniLcmApi)Api;
+        var fwDataApi = (FwDataMiniLcmApi)BaseApi;
         var complexFormEntry = fwDataApi.EntriesRepository.GetObject(_complexFormEntryId);
         await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Add ComplexFormEntryRef",
             "Remove ComplexFormEntryRef",
@@ -69,7 +69,7 @@ public async Task DuplicateComplexFormComponents_BothAreRemoved()
 
     private async Task AddDuplicateComponent()
     {
-        var fwDataApi = (FwDataMiniLcmApi)Api;
+        var fwDataApi = (FwDataMiniLcmApi)BaseApi;
         var complexFormEntry = fwDataApi.EntriesRepository.GetObject(_complexFormEntryId);
         var componentEntry = fwDataApi.EntriesRepository.GetObject(_componentEntryId);
         await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Add ComplexFormEntryRef",
@@ -107,7 +107,7 @@ public async Task DuplicateComplexFormTypes_BothAreRemoved()
 
     private async Task AddDuplicateFormType()
     {
-        var fwDataApi = (FwDataMiniLcmApi)Api;
+        var fwDataApi = (FwDataMiniLcmApi)BaseApi;
         var complexFormEntry = fwDataApi.EntriesRepository.GetObject(_complexFormEntryId);
         var complexFormType = fwDataApi.ComplexFormTypesFlattened.First();
         await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Add ComplexFormEntryRef",
diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/CreateEntryTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/CreateEntryTests.cs
@@ -17,7 +17,7 @@ protected override Task<IMiniLcmApi> NewApi()
     public async Task CreateEntry_SetsMorphType()
     {
         var entry = await Api.CreateEntry(new Entry() { LexemeForm = { ["en"] = "test" } });
-        var fwDataApi = (FwDataMiniLcmApi)Api;
+        var fwDataApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwDataApi.EntriesRepository.GetObject(entry.Id);
         lexEntry.Should().NotBeNull();
         lexEntry.LexemeFormOA.MorphTypeRA.Should().NotBeNull();
diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/MediaTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/MediaTests.cs
@@ -21,13 +21,13 @@ protected override Task<IMiniLcmApi> NewApi()
     public override async Task InitializeAsync()
     {
         await base.InitializeAsync();
-        var projectFolder = ((FwDataMiniLcmApi)Api).Cache.LangProject.LinkedFilesRootDir;
+        var projectFolder = ((FwDataMiniLcmApi)BaseApi).Cache.LangProject.LinkedFilesRootDir;
         Directory.CreateDirectory(projectFolder);
     }
 
     public override async Task DisposeAsync()
     {
-        var projectFolder = ((FwDataMiniLcmApi)Api).Cache.ProjectId.ProjectFolder;
+        var projectFolder = ((FwDataMiniLcmApi)BaseApi).Cache.ProjectId.ProjectFolder;
         if (Directory.Exists(projectFolder)) Directory.Delete(projectFolder, true);
         await base.DisposeAsync();
     }
diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/PartOfSpeechTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/PartOfSpeechTests.cs
@@ -26,7 +26,7 @@ public async Task SetPartOfSpeech_WithNullMorphoSyntaxAnalysisRA_ToValidPos()
             ]
         });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id);
         var lexSense = lexEntry.SensesOS.First();
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set MorphoSyntaxAnalysisRA to null",
@@ -61,7 +61,7 @@ public async Task SetPartOfSpeech_WithNullMorphoSyntaxAnalysisRA_ToNull()
             ]
         });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id);
         var lexSense = lexEntry.SensesOS.First();
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set MorphoSyntaxAnalysisRA to null",
diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/QueryEntryTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/QueryEntryTests.cs
@@ -13,7 +13,7 @@ public override async Task InitializeAsync()
         await base.InitializeAsync();
         var entry = await Api.CreateEntry(new Entry { LexemeForm = new MultiString { { "en", "test" } } });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id);
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set LexemeFormOA to null",
             "Restore LexemeFormOA",
diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs
@@ -26,7 +26,7 @@ public async Task UpdateEntry_WithNullLexemeFormOA_CreatesNewLexemeForm()
             LexemeForm = new MultiString { { "en", "test" } }
         });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id);
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set LexemeFormOA to null",
             "Restore LexemeFormOA",
@@ -75,7 +75,7 @@ public async Task UpdateEntry_CanUpdateExampleSentenceTranslations_WhenNoTransla
             ]
         });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id);
         lexEntry.SensesOS[0].ExamplesOS[0].TranslationsOC.Should().BeEmpty();
 
@@ -115,7 +115,7 @@ await Api.CreateEntry(new Entry
             ]
         });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entryId);
         var senseFactory = fwApi.Cache.ServiceLocator.GetInstance<ILexSenseFactory>();
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Add subsenses to sense 1",
@@ -178,7 +178,7 @@ await Api.CreateEntry(new Entry
             ]
         });
 
-        var fwApi = (FwDataMiniLcmApi)Api;
+        var fwApi = (FwDataMiniLcmApi)BaseApi;
         var lexEntry = fwApi.EntriesRepository.GetObject(entryId);
         var senseFactory = fwApi.Cache.ServiceLocator.GetInstance<ILexSenseFactory>();
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Add subsenses to sense 1",
diff --git a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs
@@ -1,3 +1,4 @@
+using System.Text;
 using FwLiteProjectSync.Tests.Fixtures;
 using MiniLcm;
 using MiniLcm.Models;
@@ -66,6 +67,39 @@ public Task DisposeAsync()
     private readonly SyncFixture _fixture = fixture;
     protected IMiniLcmApi Api = null!;
 
+    [Fact]
+    public async Task NormalizesStringsToNFD()
+    {
+        // arrange
+        var formC = "ймыл";
+        var formD = "ймыл";
+
+        formC.Should().NotBe(formD);
+        formC.Should().Be(formC.Normalize(NormalizationForm.FormC));
+        formD.Should().Be(formD.Normalize(NormalizationForm.FormD));
+        formC.Normalize(NormalizationForm.FormD).Should().Be(formD);
+
+        var entry1Id = Guid.NewGuid();
+        await Api.CreateEntry(new() { Id = entry1Id });
+        var entry1_before_formC = new Entry() { Id = entry1Id, LexemeForm = { { "en", formC } } };
+        var entry1_after_formD = new Entry() { Id = entry1Id, LexemeForm = { { "en", formD } } };
+
+        var entry2Id = Guid.NewGuid();
+        var entry2_new_formC = new Entry() { Id = entry2Id, LexemeForm = { { "en", formC } } };
+
+        // act
+        await EntrySync.SyncWithoutComplexFormsAndComponents(
+            [entry1_before_formC],
+            [entry1_after_formD, entry2_new_formC], Api);
+
+        // assert
+        var entry1After = await Api.GetEntry(entry1Id);
+        entry1After!.LexemeForm["en"].Should().Be(formD);
+        // this fails for crdt - https://github.com/sillsdev/languageforge-lexbox/issues/2065
+        // var entry2After = await Api.GetEntry(entry2Id);
+        // entry2After!.LexemeForm["en"].Should().Be(formD);
+    }
+
     [Fact]
     public async Task CanChangeComplexFormViaSync_Components()
     {
diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs
@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using MiniLcm;
+using MiniLcm.Normalization;
 using MiniLcm.SyncHelpers;
 using MiniLcm.Validators;
 using SIL.Harmony;
diff --git a/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs b/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs
@@ -5,6 +5,7 @@
 using MiniLcm;
 using MiniLcm.Media;
 using MiniLcm.Models;
+using MiniLcm.Normalization;
 using MiniLcm.Validators;
 using MiniLcm.Wrappers;
 using Reinforced.Typings.Attributes;
@@ -17,9 +18,11 @@ public class MiniLcmJsInvokable(
     IProjectIdentifier project,
     ILogger<MiniLcmJsInvokable> logger,
     MiniLcmApiNotifyWrapperFactory notificationWrapperFactory,
-    MiniLcmApiValidationWrapperFactory validationWrapperFactory) : IDisposable
+    MiniLcmApiValidationWrapperFactory validationWrapperFactory,
+    MiniLcmApiStringNormalizationWrapperFactory normalizationWrapperFactory
+    ) : IDisposable
 {
-    private readonly IMiniLcmApi _wrappedApi = api.WrapWith([validationWrapperFactory, notificationWrapperFactory], project);
+    private readonly IMiniLcmApi _wrappedApi = api.WrapWith([normalizationWrapperFactory, validationWrapperFactory, notificationWrapperFactory], project);
 
     public record MiniLcmFeatures(bool? History, bool? Write, bool? OpenWithFlex, bool? Feedback, bool? Sync, bool? Audio);
     private bool SupportsSync => project.DataFormat == ProjectDataFormat.Harmony && api is CrdtMiniLcmApi;
diff --git a/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs b/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs
@@ -1,19 +1,23 @@
+using MiniLcm.Normalization;
 using MiniLcm.Tests.AutoFakerHelpers;
+using MiniLcm.Wrappers;
 using Soenneker.Utils.AutoBogus;
-using Soenneker.Utils.AutoBogus.Config;
 
 namespace MiniLcm.Tests;
 
 public abstract class MiniLcmTestBase : IAsyncLifetime
 {
     protected static readonly AutoFaker AutoFaker = new(AutoFakerDefault.MakeConfig(["en"], 5));
     protected IMiniLcmApi Api = null!;
+    protected IMiniLcmApi BaseApi = null!;
 
     protected abstract Task<IMiniLcmApi> NewApi();
 
     public virtual async Task InitializeAsync()
     {
-        Api = await NewApi();
+        BaseApi = await NewApi();
+        BaseApi.Should().NotBeNull();
+        Api = BaseApi.WrapWith([new MiniLcmApiStringNormalizationWrapperFactory()], null!);
         Api.Should().NotBeNull();
     }
 
diff --git a/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs b/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs
@@ -1,4 +1,4 @@
-using MiniLcm.Validators;
+using MiniLcm.Normalization;
 using Moq;
 
 namespace MiniLcm.Tests;
diff --git a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs
@@ -330,25 +330,36 @@ public async Task CanFilterToExampleSentenceWithMissingSentence()
     }
 
     [Theory]
-    [InlineData("a", "a")]
-    [InlineData("a", "A")]
-    [InlineData("A", "Ã")]
-    [InlineData("ap", "apple")]
-    [InlineData("ap", "APPLE")]
-    [InlineData("ing", "walking")]
-    [InlineData("ing", "WALKING")]
-    [InlineData("Ãp", "Ãpple")]
-    [InlineData("Ãp", "ãpple")]
-    [InlineData("ap", "Ãpple")]
-    [InlineData("app", "Ãpple")]//crdt fts only kicks in at 3 chars
-    public async Task SuccessfulMatches(string searchTerm, string word)
-    {
+    [InlineData("a", "a", true)]
+    [InlineData("a", "A", false)]
+    [InlineData("A", "Ã", false)]
+    [InlineData("ap", "apple", false)]
+    [InlineData("ap", "APPLE", false)]
+    [InlineData("ing", "walking", false)]
+    [InlineData("ing", "WALKING", false)]
+    [InlineData("Ãp", "Ãpple", false)]
+    [InlineData("Ãp", "ãpple", false)]
+    [InlineData("ap", "Ãpple", false)]
+    [InlineData("app", "Ãpple", false)]//crdt fts only kicks in at 3 chars
+    [InlineData("й", "й", false)] // D, C
+    [InlineData("й", "й", false)] // C, D
+    [InlineData("й", "й", true)] // C, C
+    [InlineData("й", "й", true)] // D, D
+    [InlineData("ймыл", "ймыл", false)] // D, C
+    [InlineData("ймыл", "ймыл", false)] // C, D
+    [InlineData("ймыл", "ймыл", true)] // C, C
+    [InlineData("ймыл", "ймыл", true)] // D, D
+    public async Task SuccessfulMatches(string searchTerm, string word, bool identical)
+    {
+        // identical is to make the test cases more readable when they only differ in their normalization
+        (searchTerm == word).Should().Be(identical);
+        // remove next line in https://github.com/sillsdev/languageforge-lexbox/issues/2065
         word = word.Normalize(NormalizationForm.FormD);
-        //should we be normalizing the search term internally?
-        searchTerm = searchTerm.Normalize(NormalizationForm.FormD);
         await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } });
         var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync();
-        words.Should().Contain(word);
+        words.Should().NotBeEmpty();
+        // Like LicLCM the MiniLcm API should normalize to NFD
+        words.Should().Contain(word.Normalize(NormalizationForm.FormD));
     }
 
     [Theory]
diff --git a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs
@@ -1,10 +1,8 @@
 using System.Text;
-using MiniLcm;
 using MiniLcm.Models;
-using MiniLcm.SyncHelpers;
 using MiniLcm.Wrappers;
 
-namespace MiniLcm.Validators;
+namespace MiniLcm.Normalization;
 
 public class MiniLcmApiStringNormalizationWrapperFactory() : IMiniLcmWrapperFactory
 {
diff --git a/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs b/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs
@@ -1,6 +1,7 @@
 using FluentValidation;
 using Microsoft.Extensions.DependencyInjection;
 using MiniLcm.Models;
+using MiniLcm.Normalization;
 
 namespace MiniLcm.Validators;
 

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ protected override Task<IMiniLcmApi> NewApi()`
`17`	`17`	`public async Task CreateEntry_SetsMorphType()`
`18`	`18`	`{`
`19`	`19`	`var entry = await Api.CreateEntry(new Entry() { LexemeForm = { ["en"] = "test" } });`
`20`		`- var fwDataApi = (FwDataMiniLcmApi)Api;`
	`20`	`+ var fwDataApi = (FwDataMiniLcmApi)BaseApi;`
`21`	`21`	`var lexEntry = fwDataApi.EntriesRepository.GetObject(entry.Id);`
`22`	`22`	`lexEntry.Should().NotBeNull();`
`23`	`23`	`lexEntry.LexemeFormOA.MorphTypeRA.Should().NotBeNull();`
Original file line number	Diff line number	Diff line change
`@@ -21,13 +21,13 @@ protected override Task<IMiniLcmApi> NewApi()`
`21`	`21`	`public override async Task InitializeAsync()`
`22`	`22`	`{`
`23`	`23`	`await base.InitializeAsync();`
`24`		`- var projectFolder = ((FwDataMiniLcmApi)Api).Cache.LangProject.LinkedFilesRootDir;`
	`24`	`+ var projectFolder = ((FwDataMiniLcmApi)BaseApi).Cache.LangProject.LinkedFilesRootDir;`
`25`	`25`	`Directory.CreateDirectory(projectFolder);`
`26`	`26`	`}`
`27`	`27`
`28`	`28`	`public override async Task DisposeAsync()`
`29`	`29`	`{`
`30`		`- var projectFolder = ((FwDataMiniLcmApi)Api).Cache.ProjectId.ProjectFolder;`
	`30`	`+ var projectFolder = ((FwDataMiniLcmApi)BaseApi).Cache.ProjectId.ProjectFolder;`
`31`	`31`	`if (Directory.Exists(projectFolder)) Directory.Delete(projectFolder, true);`
`32`	`32`	`await base.DisposeAsync();`
`33`	`33`	`}`