diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/BasicApiTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/BasicApiTests.cs index c6e8210468..163cca0d63 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/BasicApiTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/BasicApiTests.cs @@ -22,7 +22,7 @@ public async Task GetEntry_WithNullLexemeFormOA_CreatesNewLexemeForm() LexemeForm = new MultiString { { "en", "test" } } }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id); UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set LexemeFormOA to null", "Restore LexemeFormOA", diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/ComplexFormComponentTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/ComplexFormComponentTests.cs index 96574994ed..ca0634c511 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/ComplexFormComponentTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/ComplexFormComponentTests.cs @@ -27,7 +27,7 @@ protected override Task NewApi() public override async Task InitializeAsync() { await base.InitializeAsync(); - var fwDataApi = (FwDataMiniLcmApi)Api; + var fwDataApi = (FwDataMiniLcmApi)BaseApi; var complexFormEntry = fwDataApi.EntriesRepository.GetObject(_complexFormEntryId); await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Add ComplexFormEntryRef", "Remove ComplexFormEntryRef", @@ -69,7 +69,7 @@ public async Task DuplicateComplexFormComponents_BothAreRemoved() private async Task AddDuplicateComponent() { - var fwDataApi = (FwDataMiniLcmApi)Api; + var fwDataApi = (FwDataMiniLcmApi)BaseApi; var complexFormEntry = fwDataApi.EntriesRepository.GetObject(_complexFormEntryId); var componentEntry = fwDataApi.EntriesRepository.GetObject(_componentEntryId); await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Add ComplexFormEntryRef", @@ -107,7 +107,7 @@ public async Task DuplicateComplexFormTypes_BothAreRemoved() private async Task AddDuplicateFormType() { - var fwDataApi = (FwDataMiniLcmApi)Api; + var fwDataApi = (FwDataMiniLcmApi)BaseApi; var complexFormEntry = fwDataApi.EntriesRepository.GetObject(_complexFormEntryId); var complexFormType = fwDataApi.ComplexFormTypesFlattened.First(); await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Add ComplexFormEntryRef", diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/CreateEntryTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/CreateEntryTests.cs index 33bbb186cf..3b801ddadf 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/CreateEntryTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/CreateEntryTests.cs @@ -17,7 +17,7 @@ protected override Task NewApi() public async Task CreateEntry_SetsMorphType() { var entry = await Api.CreateEntry(new Entry() { LexemeForm = { ["en"] = "test" } }); - var fwDataApi = (FwDataMiniLcmApi)Api; + var fwDataApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwDataApi.EntriesRepository.GetObject(entry.Id); lexEntry.Should().NotBeNull(); lexEntry.LexemeFormOA.MorphTypeRA.Should().NotBeNull(); diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/MediaTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/MediaTests.cs index b56ac52bda..4f7f9ea077 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/MediaTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/MediaTests.cs @@ -21,13 +21,13 @@ protected override Task NewApi() public override async Task InitializeAsync() { await base.InitializeAsync(); - var projectFolder = ((FwDataMiniLcmApi)Api).Cache.LangProject.LinkedFilesRootDir; + var projectFolder = ((FwDataMiniLcmApi)BaseApi).Cache.LangProject.LinkedFilesRootDir; Directory.CreateDirectory(projectFolder); } public override async Task DisposeAsync() { - var projectFolder = ((FwDataMiniLcmApi)Api).Cache.ProjectId.ProjectFolder; + var projectFolder = ((FwDataMiniLcmApi)BaseApi).Cache.ProjectId.ProjectFolder; if (Directory.Exists(projectFolder)) Directory.Delete(projectFolder, true); await base.DisposeAsync(); } diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/PartOfSpeechTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/PartOfSpeechTests.cs index 055b650a15..3943623a06 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/PartOfSpeechTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/PartOfSpeechTests.cs @@ -26,7 +26,7 @@ public async Task SetPartOfSpeech_WithNullMorphoSyntaxAnalysisRA_ToValidPos() ] }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id); var lexSense = lexEntry.SensesOS.First(); UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set MorphoSyntaxAnalysisRA to null", @@ -61,7 +61,7 @@ public async Task SetPartOfSpeech_WithNullMorphoSyntaxAnalysisRA_ToNull() ] }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id); var lexSense = lexEntry.SensesOS.First(); UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set MorphoSyntaxAnalysisRA to null", diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/QueryEntryTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/QueryEntryTests.cs index 1742d07e25..6bc0e4be51 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/QueryEntryTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/QueryEntryTests.cs @@ -13,7 +13,7 @@ public override async Task InitializeAsync() await base.InitializeAsync(); var entry = await Api.CreateEntry(new Entry { LexemeForm = new MultiString { { "en", "test" } } }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id); UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set LexemeFormOA to null", "Restore LexemeFormOA", diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs index bac3d6d5e3..1e184836b1 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/UpdateEntryTests.cs @@ -26,7 +26,7 @@ public async Task UpdateEntry_WithNullLexemeFormOA_CreatesNewLexemeForm() LexemeForm = new MultiString { { "en", "test" } } }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id); UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Set LexemeFormOA to null", "Restore LexemeFormOA", @@ -75,7 +75,7 @@ public async Task UpdateEntry_CanUpdateExampleSentenceTranslations_WhenNoTransla ] }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entry.Id); lexEntry.SensesOS[0].ExamplesOS[0].TranslationsOC.Should().BeEmpty(); @@ -115,7 +115,7 @@ await Api.CreateEntry(new Entry ] }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entryId); var senseFactory = fwApi.Cache.ServiceLocator.GetInstance(); UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Add subsenses to sense 1", @@ -178,7 +178,7 @@ await Api.CreateEntry(new Entry ] }); - var fwApi = (FwDataMiniLcmApi)Api; + var fwApi = (FwDataMiniLcmApi)BaseApi; var lexEntry = fwApi.EntriesRepository.GetObject(entryId); var senseFactory = fwApi.Cache.ServiceLocator.GetInstance(); UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Add subsenses to sense 1", diff --git a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs index fcbafdc2a9..fc45494c15 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/EntrySyncTests.cs @@ -1,3 +1,4 @@ +using System.Text; using FwLiteProjectSync.Tests.Fixtures; using MiniLcm; using MiniLcm.Models; @@ -66,6 +67,39 @@ public Task DisposeAsync() private readonly SyncFixture _fixture = fixture; protected IMiniLcmApi Api = null!; + [Fact] + public async Task NormalizesStringsToNFD() + { + // arrange + var formC = "ймыл"; + var formD = "ймыл"; + + formC.Should().NotBe(formD); + formC.Should().Be(formC.Normalize(NormalizationForm.FormC)); + formD.Should().Be(formD.Normalize(NormalizationForm.FormD)); + formC.Normalize(NormalizationForm.FormD).Should().Be(formD); + + var entry1Id = Guid.NewGuid(); + await Api.CreateEntry(new() { Id = entry1Id }); + var entry1_before_formC = new Entry() { Id = entry1Id, LexemeForm = { { "en", formC } } }; + var entry1_after_formD = new Entry() { Id = entry1Id, LexemeForm = { { "en", formD } } }; + + var entry2Id = Guid.NewGuid(); + var entry2_new_formC = new Entry() { Id = entry2Id, LexemeForm = { { "en", formC } } }; + + // act + await EntrySync.SyncWithoutComplexFormsAndComponents( + [entry1_before_formC], + [entry1_after_formD, entry2_new_formC], Api); + + // assert + var entry1After = await Api.GetEntry(entry1Id); + entry1After!.LexemeForm["en"].Should().Be(formD); + // this fails for crdt - https://github.com/sillsdev/languageforge-lexbox/issues/2065 + // var entry2After = await Api.GetEntry(entry2Id); + // entry2After!.LexemeForm["en"].Should().Be(formD); + } + [Fact] public async Task CanChangeComplexFormViaSync_Components() { diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 01fe941680..790cdfde04 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MiniLcm; +using MiniLcm.Normalization; using MiniLcm.SyncHelpers; using MiniLcm.Validators; using SIL.Harmony; diff --git a/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs b/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs index 3126be09dd..3daf61bddd 100644 --- a/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs +++ b/backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs @@ -5,6 +5,7 @@ using MiniLcm; using MiniLcm.Media; using MiniLcm.Models; +using MiniLcm.Normalization; using MiniLcm.Validators; using MiniLcm.Wrappers; using Reinforced.Typings.Attributes; @@ -17,9 +18,11 @@ public class MiniLcmJsInvokable( IProjectIdentifier project, ILogger logger, MiniLcmApiNotifyWrapperFactory notificationWrapperFactory, - MiniLcmApiValidationWrapperFactory validationWrapperFactory) : IDisposable + MiniLcmApiValidationWrapperFactory validationWrapperFactory, + MiniLcmApiStringNormalizationWrapperFactory normalizationWrapperFactory + ) : IDisposable { - private readonly IMiniLcmApi _wrappedApi = api.WrapWith([validationWrapperFactory, notificationWrapperFactory], project); + private readonly IMiniLcmApi _wrappedApi = api.WrapWith([normalizationWrapperFactory, validationWrapperFactory, notificationWrapperFactory], project); public record MiniLcmFeatures(bool? History, bool? Write, bool? OpenWithFlex, bool? Feedback, bool? Sync, bool? Audio); private bool SupportsSync => project.DataFormat == ProjectDataFormat.Harmony && api is CrdtMiniLcmApi; diff --git a/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs b/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs index 37ee6f0763..1b3d255600 100644 --- a/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs +++ b/backend/FwLite/MiniLcm.Tests/MiniLcmTestBase.cs @@ -1,6 +1,7 @@ +using MiniLcm.Normalization; using MiniLcm.Tests.AutoFakerHelpers; +using MiniLcm.Wrappers; using Soenneker.Utils.AutoBogus; -using Soenneker.Utils.AutoBogus.Config; namespace MiniLcm.Tests; @@ -8,12 +9,15 @@ public abstract class MiniLcmTestBase : IAsyncLifetime { protected static readonly AutoFaker AutoFaker = new(AutoFakerDefault.MakeConfig(["en"], 5)); protected IMiniLcmApi Api = null!; + protected IMiniLcmApi BaseApi = null!; protected abstract Task NewApi(); public virtual async Task InitializeAsync() { - Api = await NewApi(); + BaseApi = await NewApi(); + BaseApi.Should().NotBeNull(); + Api = BaseApi.WrapWith([new MiniLcmApiStringNormalizationWrapperFactory()], null!); Api.Should().NotBeNull(); } diff --git a/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs b/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs index 4739da9198..4feff444c8 100644 --- a/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs +++ b/backend/FwLite/MiniLcm.Tests/NormalizationTests.cs @@ -1,4 +1,4 @@ -using MiniLcm.Validators; +using MiniLcm.Normalization; using Moq; namespace MiniLcm.Tests; diff --git a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs index ce4f0fb8f9..2916504497 100644 --- a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs @@ -330,25 +330,36 @@ public async Task CanFilterToExampleSentenceWithMissingSentence() } [Theory] - [InlineData("a", "a")] - [InlineData("a", "A")] - [InlineData("A", "Ã")] - [InlineData("ap", "apple")] - [InlineData("ap", "APPLE")] - [InlineData("ing", "walking")] - [InlineData("ing", "WALKING")] - [InlineData("Ãp", "Ãpple")] - [InlineData("Ãp", "ãpple")] - [InlineData("ap", "Ãpple")] - [InlineData("app", "Ãpple")]//crdt fts only kicks in at 3 chars - public async Task SuccessfulMatches(string searchTerm, string word) - { + [InlineData("a", "a", true)] + [InlineData("a", "A", false)] + [InlineData("A", "Ã", false)] + [InlineData("ap", "apple", false)] + [InlineData("ap", "APPLE", false)] + [InlineData("ing", "walking", false)] + [InlineData("ing", "WALKING", false)] + [InlineData("Ãp", "Ãpple", false)] + [InlineData("Ãp", "ãpple", false)] + [InlineData("ap", "Ãpple", false)] + [InlineData("app", "Ãpple", false)]//crdt fts only kicks in at 3 chars + [InlineData("й", "й", false)] // D, C + [InlineData("й", "й", false)] // C, D + [InlineData("й", "й", true)] // C, C + [InlineData("й", "й", true)] // D, D + [InlineData("ймыл", "ймыл", false)] // D, C + [InlineData("ймыл", "ймыл", false)] // C, D + [InlineData("ймыл", "ймыл", true)] // C, C + [InlineData("ймыл", "ймыл", true)] // D, D + public async Task SuccessfulMatches(string searchTerm, string word, bool identical) + { + // identical is to make the test cases more readable when they only differ in their normalization + (searchTerm == word).Should().Be(identical); + // remove next line in https://github.com/sillsdev/languageforge-lexbox/issues/2065 word = word.Normalize(NormalizationForm.FormD); - //should we be normalizing the search term internally? - searchTerm = searchTerm.Normalize(NormalizationForm.FormD); await Api.CreateEntry(new Entry { LexemeForm = { ["en"] = word } }); var words = await Api.SearchEntries(searchTerm).Select(e => e.LexemeForm["en"]).ToArrayAsync(); - words.Should().Contain(word); + words.Should().NotBeEmpty(); + // Like LicLCM the MiniLcm API should normalize to NFD + words.Should().Contain(word.Normalize(NormalizationForm.FormD)); } [Theory] diff --git a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs index 3041eff940..1204dc2a11 100644 --- a/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs +++ b/backend/FwLite/MiniLcm/Normalization/MiniLcmApiStringNormalizationWrapper.cs @@ -1,10 +1,8 @@ using System.Text; -using MiniLcm; using MiniLcm.Models; -using MiniLcm.SyncHelpers; using MiniLcm.Wrappers; -namespace MiniLcm.Validators; +namespace MiniLcm.Normalization; public class MiniLcmApiStringNormalizationWrapperFactory() : IMiniLcmWrapperFactory { diff --git a/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs b/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs index f74ca29fb3..224bfd3c9e 100644 --- a/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs +++ b/backend/FwLite/MiniLcm/Validators/MiniLcmValidators.cs @@ -1,6 +1,7 @@ using FluentValidation; using Microsoft.Extensions.DependencyInjection; using MiniLcm.Models; +using MiniLcm.Normalization; namespace MiniLcm.Validators;