From 5e574e41ad72f2d057b58f945cef29ca9a014df4 Mon Sep 17 00:00:00 2001 From: Christopher Thonfeld-Guckes Date: Mon, 11 Dec 2023 15:43:04 +0100 Subject: [PATCH] fix: sorting and properties --- .github/workflows/docker-build.yml | 2 +- src/WebIO/WebIO.Elastic/Data/IndexedDevice.cs | 10 +- .../WebIO.Elastic/Data/IndexedInterface.cs | 12 +- src/WebIO/WebIO.Elastic/Data/IndexedStream.cs | 22 ++-- .../Hosting/HostingExtensions.cs | 2 +- .../Management/Search/SearchRequest.cs | 1 + .../Management/Search/Searcher.cs | 24 +++- .../Management/Search/TextFieldSelector.cs | 3 +- src/WebIO/WebIO.Elastic/Management/Util.cs | 5 +- .../IntegrationTests/ElasticTest.cs | 16 +-- .../DataAccess/Elastic/DeviceSearcher.cs | 28 +++++ .../Elastic/ElasticDeviceRepository.cs | 17 +++ .../DataAccess/Elastic/InterfaceSearcher.cs | 77 +++++++++++- .../DataAccess/Elastic/StreamSearcher.cs | 115 ++++++++++++++++-- 14 files changed, 292 insertions(+), 42 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 55dfe1b..2e6adbd 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -135,7 +135,7 @@ jobs: sed -i "s@#AZURE_CLIENT_ID#@${{ secrets.AZURE_CLIENT_ID }}@g" ./build/app/appsettings.Production.json sed -i "s@#AZURE_TENANT_ID#@${{ secrets.AZURE_TENANT_ID }}@g" ./build/app/appsettings.Production.json sed -i "s@#AZURE_AUDIENCE#@${{ secrets.AZURE_AUDIENCE }}@g" ./build/app/appsettings.Production.json - sed -i "s@#AZURE_INSTANCE#@${{ env.AZURE_INSTANCE }}@g" ./build/app/appsettings.Production.json + sed -i "s@#AZURE_INSTANCE#@https://login.microsoftonline.com/@g" ./build/app/appsettings.Production.json - name: Docker build run: | cd build diff --git a/src/WebIO/WebIO.Elastic/Data/IndexedDevice.cs b/src/WebIO/WebIO.Elastic/Data/IndexedDevice.cs index 123332a..c9368b9 100644 --- a/src/WebIO/WebIO.Elastic/Data/IndexedDevice.cs +++ b/src/WebIO/WebIO.Elastic/Data/IndexedDevice.cs @@ -2,13 +2,17 @@ using System.Collections.Immutable; using Management; +using Nest; public record IndexedDevice : IIndexedEntity { public Guid Id { get; init; } = Guid.NewGuid(); - public string Name { get; init; } = string.Empty; - public string DeviceType { get; init; } = string.Empty; + [Keyword] public string Name { get; init; } = string.Empty; + [Keyword] public string DeviceType { get; init; } = string.Empty; public string Comment { get; init; } = string.Empty; + + [Nested] public IReadOnlyDictionary Properties { get; init; } = ImmutableDictionary.Empty; + public int InterfaceCount { get; init; } -} \ No newline at end of file +} diff --git a/src/WebIO/WebIO.Elastic/Data/IndexedInterface.cs b/src/WebIO/WebIO.Elastic/Data/IndexedInterface.cs index 6a113d2..deee6da 100644 --- a/src/WebIO/WebIO.Elastic/Data/IndexedInterface.cs +++ b/src/WebIO/WebIO.Elastic/Data/IndexedInterface.cs @@ -2,18 +2,27 @@ namespace WebIO.Elastic.Data; using System.Collections.Immutable; using Management; +using Nest; public record IndexedInterface : IIndexedEntity { public Guid Id { get; init; } = Guid.NewGuid(); + [Keyword] public Guid DeviceId { get; init; } + [Keyword] public string Name { get; init; } = string.Empty; public int Index { get; init; } + [Keyword] public string InterfaceTemplate { get; init; } = string.Empty; public string Comment { get; init; } = string.Empty; + [Keyword] public string DeviceType { get; init; } = string.Empty; + [Keyword] public string DeviceName { get; init; } = string.Empty; + + [Nested] public IReadOnlyDictionary Properties { get; init; } = ImmutableDictionary.Empty; + public int StreamsCountVideoSend { get; init; } public int StreamsCountAudioSend { get; init; } public int StreamsCountAncillarySend { get; init; } @@ -21,6 +30,7 @@ public record IndexedInterface : IIndexedEntity public int StreamsCountAudioReceive { get; init; } public int StreamsCountAncillaryReceive { get; init; } + [Nested] public IReadOnlyDictionary DeviceProperties { get; init; } = ImmutableDictionary.Empty; -} \ No newline at end of file +} diff --git a/src/WebIO/WebIO.Elastic/Data/IndexedStream.cs b/src/WebIO/WebIO.Elastic/Data/IndexedStream.cs index 2a5d272..95f9604 100644 --- a/src/WebIO/WebIO.Elastic/Data/IndexedStream.cs +++ b/src/WebIO/WebIO.Elastic/Data/IndexedStream.cs @@ -2,24 +2,30 @@ namespace WebIO.Elastic.Data; using System.Collections.Immutable; using Management; +using Nest; public record IndexedStream : IIndexedEntity { public Guid Id { get; init; } = Guid.NewGuid(); - public Guid InterfaceId { get; init; } - public string DeviceName { get; init; } = string.Empty; - public string InterfaceName { get; init; } = string.Empty; - public string Name { get; init; } = string.Empty; + [Keyword] public Guid InterfaceId { get; init; } + [Keyword] public string DeviceName { get; init; } = string.Empty; + [Keyword] public string InterfaceName { get; init; } = string.Empty; + [Keyword] public string Name { get; init; } = string.Empty; public string Comment { get; init; } = string.Empty; - public StreamType Type { get; init; } - public StreamDirection Direction { get; init; } + [Keyword] public StreamType Type { get; init; } + [Keyword] public StreamDirection Direction { get; init; } + + [Nested] public IReadOnlyDictionary Properties { get; init; } = ImmutableDictionary.Empty; - public Guid DeviceId { get; init; } - public string DeviceType { get; init; } = string.Empty; + [Keyword] public Guid DeviceId { get; init; } + [Keyword] public string DeviceType { get; init; } = string.Empty; + + [Nested] public IReadOnlyDictionary InterfaceProperties { get; init; } = ImmutableDictionary.Empty; + [Nested] public IReadOnlyDictionary DeviceProperties { get; init; } = ImmutableDictionary.Empty; } diff --git a/src/WebIO/WebIO.Elastic/Hosting/HostingExtensions.cs b/src/WebIO/WebIO.Elastic/Hosting/HostingExtensions.cs index b1b00f1..25deae6 100644 --- a/src/WebIO/WebIO.Elastic/Hosting/HostingExtensions.cs +++ b/src/WebIO/WebIO.Elastic/Hosting/HostingExtensions.cs @@ -21,7 +21,7 @@ public static IServiceCollection RegisterElastic( builder.AddSingleton(config); var connectionSettings = new ConnectionSettings(config.Url); - // connectionSettings.EnableDebugMode(); + connectionSettings.EnableDebugMode(); if (config.Proxy != null) { connectionSettings.Proxy(config.Proxy, config.ProxyUser, config.ProxyPassword); diff --git a/src/WebIO/WebIO.Elastic/Management/Search/SearchRequest.cs b/src/WebIO/WebIO.Elastic/Management/Search/SearchRequest.cs index a6e24cc..3cd8b5e 100644 --- a/src/WebIO/WebIO.Elastic/Management/Search/SearchRequest.cs +++ b/src/WebIO/WebIO.Elastic/Management/Search/SearchRequest.cs @@ -4,4 +4,5 @@ public record SearchRequest { public int? Take { get; init; } public string? Fulltext { get; init; } + public IEnumerable> Sorting { get; init; } = Enumerable.Empty>(); } diff --git a/src/WebIO/WebIO.Elastic/Management/Search/Searcher.cs b/src/WebIO/WebIO.Elastic/Management/Search/Searcher.cs index cbe8adf..a7333b9 100644 --- a/src/WebIO/WebIO.Elastic/Management/Search/Searcher.cs +++ b/src/WebIO/WebIO.Elastic/Management/Search/Searcher.cs @@ -1,6 +1,8 @@ namespace WebIO.Elastic.Management.Search; using System.Runtime.CompilerServices; +using System.Text; +using Elasticsearch.Net; using IndexManagement; using Microsoft.Extensions.Logging; using Nest; @@ -28,12 +30,12 @@ protected Searcher( public async Task> FindAllAsync(TSearchRequest request, CancellationToken ct) { var searchResponse = await _client.SearchAsync(sd => PrepareQuery(request, sd), ct); - // var json = Encoding.UTF8.GetString(searchResponse.ApiCall.RequestBodyInBytes); + var json = Encoding.UTF8.GetString(searchResponse.ApiCall.RequestBodyInBytes); return new(Documents: IterateResultsAsync(searchResponse, request, ct), Total: searchResponse.Total); } public async Task GetAsync(TId id, CancellationToken ct) - => (await GetAllAsync(new[] {id}, ct)).FirstOrDefault(); + => (await GetAllAsync(new[] { id }, ct)).FirstOrDefault(); public async Task> GetAllAsync(IEnumerable ids, CancellationToken ct) => (await _client.GetManyAsync(ids.Select(id => $"{id}"), _indexManager.GetAliasName(), ct)).Select(hit @@ -70,11 +72,13 @@ private SearchDescriptor PrepareQuery( TSearchRequest request, SearchDescriptor sd, int currentPosition = 0) - => UseIndex(_indexManager.GetAliasName(), + => UseIndex( + _indexManager.GetAliasName(), + Sort(request, SkipTake( currentPosition, request.Take ?? _config.BatchSize, - ToQuery(request)(sd))); + ToQuery(request)(sd)))); private static SearchDescriptor UseIndex(string index, SearchDescriptor sd) { @@ -91,4 +95,16 @@ private static SearchDescriptor SkipTake( sd.Take(take); return sd; } + + private static SearchDescriptor Sort( + SearchRequest request, + SearchDescriptor sd) + { + sd.Sort(d => request.Sorting.Aggregate(d, + (sort, kv) => sort.Field(f + => f.Field(kv.Key) + .Order(kv.Value == "asc" ? SortOrder.Ascending : SortOrder.Descending) + .UnmappedType(FieldType.Keyword)))); + return sd; + } } diff --git a/src/WebIO/WebIO.Elastic/Management/Search/TextFieldSelector.cs b/src/WebIO/WebIO.Elastic/Management/Search/TextFieldSelector.cs index b37cbca..da16fb1 100644 --- a/src/WebIO/WebIO.Elastic/Management/Search/TextFieldSelector.cs +++ b/src/WebIO/WebIO.Elastic/Management/Search/TextFieldSelector.cs @@ -4,7 +4,8 @@ namespace WebIO.Elastic.Management.Search; public record TextFieldSelector { - public Expression> Selector { get; init; } = null!; + public string Name { get; init; } = string.Empty; + public Expression>? Selector { get; init; } public double Boost { get; init; } public TextFieldType Type { get; init; } } diff --git a/src/WebIO/WebIO.Elastic/Management/Util.cs b/src/WebIO/WebIO.Elastic/Management/Util.cs index e39709d..2d707ed 100644 --- a/src/WebIO/WebIO.Elastic/Management/Util.cs +++ b/src/WebIO/WebIO.Elastic/Management/Util.cs @@ -176,7 +176,10 @@ private static Func, FieldsDescriptor().Database.EnsureCreated(); } - [Fact(Skip = "Broken atm")] + [Fact] public void WriteAndLoadEntity() { var repo = _app.Services.GetRequiredService(); diff --git a/src/WebIO/WebIO/DataAccess/Elastic/DeviceSearcher.cs b/src/WebIO/WebIO/DataAccess/Elastic/DeviceSearcher.cs index 38b7c2b..b73dca2 100644 --- a/src/WebIO/WebIO/DataAccess/Elastic/DeviceSearcher.cs +++ b/src/WebIO/WebIO/DataAccess/Elastic/DeviceSearcher.cs @@ -45,6 +45,34 @@ protected override Func, SearchDescriptor Util.ToTextQuery(mf, request.DeviceName, fields)); } + foreach (var (key, value) in request.Properties) + { + if (!string.IsNullOrWhiteSpace(value)) + { + var fields = new[] + { + new TextFieldSelector + { + Name = $"properties.{key}", + Boost = 10, + Type = TextFieldType.Exact, + }, + new TextFieldSelector + { + Name = $"properties.{key}", + Boost = 8, + Type = TextFieldType.PrefixWildcard, + }, + }; + + mustFilters.Add(mf + => mf.Nested(nqd + => nqd + .Path(d => d.Properties) + .Query(q => Util.ToTextQuery(q, value, fields)))); + } + } + return sd => sd .MinScore(mustFilters.Any() ? 8 : 0) diff --git a/src/WebIO/WebIO/DataAccess/Elastic/ElasticDeviceRepository.cs b/src/WebIO/WebIO/DataAccess/Elastic/ElasticDeviceRepository.cs index 1bb61b8..c5cd4a3 100644 --- a/src/WebIO/WebIO/DataAccess/Elastic/ElasticDeviceRepository.cs +++ b/src/WebIO/WebIO/DataAccess/Elastic/ElasticDeviceRepository.cs @@ -147,6 +147,7 @@ private SearchResult FindDeviceByQuery(Query query) DeviceName = query.Filter.FirstOrDefault(f => f.Key == "Name")?.Value ?? string.Empty, Properties = query.Filter.ToDictionary(i => i.Key, i => i.Value), + Sorting = ToSortDictionary(query), }, default).GetAwaiter().GetResult(); private SearchResult FindInterfaceByQuery(Query query) @@ -162,6 +163,7 @@ private SearchResult FindInterfaceByQuery(Query query) string.Empty, DeviceName = query.Filter.FirstOrDefault(f => f.Key == nameof(InterfaceSearchRequest.DeviceName))?.Value ?? string.Empty, + Sorting = ToSortDictionary(query), Properties = query.Filter.ToDictionary(i => i.Key, i => i.Value), }, default).GetAwaiter().GetResult(); } @@ -177,6 +179,7 @@ private SearchResult FindStreamByQuery(Query query) string.Empty, DeviceName = query.Filter.FirstOrDefault(f => f.Key == nameof(StreamSearchRequest.DeviceName))?.Value ?? string.Empty, + Sorting = ToSortDictionary(query), Properties = query.Filter.ToDictionary(i => i.Key, i => i.Value), }, default).GetAwaiter().GetResult(); @@ -314,6 +317,20 @@ private static StreamInfo ToStreamInfo(IndexedStream stream) stream.DeviceProperties.ToImmutableDictionary(), stream.InterfaceProperties.ToImmutableDictionary(), null); + + private static IEnumerable> ToSortDictionary(Query query) + => query.Sort?.Split(',').Select(LcFirstIfBaseObjectMember) + .Zip(query.Order?.Split(',') ?? Enumerable.Empty(), KeyValuePair.Create) + ?? ImmutableArray>.Empty; + + private static string LcFirstIfBaseObjectMember(string field) + => field switch { + "Name" => "name", + "DeviceName" => "deviceName", + "InterfaceName" => "interfaceName", + "StreamName" => "streamName", + _ => $"properties.{field}", + }; } public record DeviceSearchRequest : SearchRequest diff --git a/src/WebIO/WebIO/DataAccess/Elastic/InterfaceSearcher.cs b/src/WebIO/WebIO/DataAccess/Elastic/InterfaceSearcher.cs index 2bb6db5..8e88320 100644 --- a/src/WebIO/WebIO/DataAccess/Elastic/InterfaceSearcher.cs +++ b/src/WebIO/WebIO/DataAccess/Elastic/InterfaceSearcher.cs @@ -23,15 +23,12 @@ public InterfaceSearcher( protected override Func, SearchDescriptor> ToQuery( InterfaceSearchRequest request) { - var mustFilters = new List, QueryContainer>>(); - if (request.DeviceId != null && request.DeviceId != Guid.Empty) { - mustFilters.Add(mf - => mf.Match(f - => f.Field(d => d.DeviceId).Query($"{request.DeviceId}"))); + return FilterByDeviceId(request); } + var mustFilters = new List, QueryContainer>>(); if (!string.IsNullOrWhiteSpace(request.InterfaceName)) { var fields = new[] @@ -72,11 +69,79 @@ protected override Func, SearchDescriptor Util.ToTextQuery(mf, request.DeviceName, fields)); } + var propFilters = new List, QueryContainer>>(); + foreach (var (key, value) in request.Properties) + { + if (!string.IsNullOrWhiteSpace(value)) + { + var propFields = new[] + { + new TextFieldSelector + { + Name = $"properties.{key}", + Boost = 10, + Type = TextFieldType.Exact, + }, + new TextFieldSelector + { + Name = $"properties.{key}", + Boost = 8, + Type = TextFieldType.PrefixWildcard, + }, + }; + + propFilters.Add(mf + => mf.Nested(nqd + => nqd + .Path(d => d.Properties) + .Query(q => Util.ToTextQuery(q, value, propFields)))); + } + } + + foreach (var (key, value) in request.Properties) + { + if (!string.IsNullOrWhiteSpace(value)) + { + var fields = new[] + { + new TextFieldSelector + { + Name = $"deviceProperties.{key}", + Boost = 10, + Type = TextFieldType.Exact, + }, + new TextFieldSelector + { + Name = $"deviceProperties.{key}", + Boost = 8, + Type = TextFieldType.PrefixWildcard, + }, + }; + + propFilters.Add(mf + => mf.Nested(nqd + => nqd + .Path(d => d.DeviceProperties) + .Query(q => Util.ToTextQuery(q, value, fields)))); + } + } + return sd => sd .MinScore(mustFilters.Any() ? 8 : 0) .Query(qd => qd.Bool(bqd - => bqd.Must(mustFilters.ToArray()))); + => bqd.Must(mustFilters.ToArray()) + .Should(propFilters))); } + + private static Func, SearchDescriptor> FilterByDeviceId( + InterfaceSearchRequest request) + => sd + => sd + .Query(qd + => qd.Bool(bqd + => bqd.Must(mf + => mf.Match(f + => f.Field(d => d.DeviceId).Query($"{request.DeviceId}"))))); } diff --git a/src/WebIO/WebIO/DataAccess/Elastic/StreamSearcher.cs b/src/WebIO/WebIO/DataAccess/Elastic/StreamSearcher.cs index b4e3af9..9f02cad 100644 --- a/src/WebIO/WebIO/DataAccess/Elastic/StreamSearcher.cs +++ b/src/WebIO/WebIO/DataAccess/Elastic/StreamSearcher.cs @@ -23,6 +23,11 @@ public StreamSearcher( protected override Func, SearchDescriptor> ToQuery( StreamSearchRequest request) { + if (request.InterfaceIds.Any()) + { + return FilterByInterfaceIds(request.InterfaceIds); + } + var mustFilters = new List, QueryContainer>>(); if (!string.IsNullOrWhiteSpace(request.StreamName)) @@ -44,7 +49,6 @@ protected override Func, SearchDescriptor Util.ToTextQuery(mf, request.StreamName, fields)); } - if (!string.IsNullOrWhiteSpace(request.DeviceName)) { @@ -65,7 +69,7 @@ protected override Func, SearchDescriptor Util.ToTextQuery(mf, request.DeviceName, fields)); } - + if (!string.IsNullOrWhiteSpace(request.InterfaceName)) { var fields = new[] @@ -86,17 +90,110 @@ protected override Func, SearchDescriptor Util.ToTextQuery(mf, request.InterfaceName, fields)); } + var propFilters = new List, QueryContainer>>(); + foreach (var (key, value) in request.Properties) + { + if (!string.IsNullOrWhiteSpace(value)) + { + var fields = new[] + { + new TextFieldSelector + { + Name = $"properties.{key}", + Boost = 10, + Type = TextFieldType.Exact, + }, + new TextFieldSelector + { + Name = $"properties.{key}", + Boost = 8, + Type = TextFieldType.PrefixWildcard, + }, + }; + + propFilters.Add(mf + => mf.Nested(nqd + => nqd + .Path(d => d.Properties) + .Query(q => Util.ToTextQuery(q, value, fields)))); + } + } + + foreach (var (key, value) in request.Properties) + { + if (!string.IsNullOrWhiteSpace(value)) + { + var fields = new[] + { + new TextFieldSelector + { + Name = $"interfaceProperties.{key}", + Boost = 10, + Type = TextFieldType.Exact, + }, + new TextFieldSelector + { + Name = $"interfaceProperties.{key}", + Boost = 8, + Type = TextFieldType.PrefixWildcard, + }, + }; + + propFilters.Add(mf + => mf.Nested(nqd + => nqd + .Path(d => d.InterfaceProperties) + .Query(q => Util.ToTextQuery(q, value, fields)))); + } + } + + foreach (var (key, value) in request.Properties) + { + if (!string.IsNullOrWhiteSpace(value)) + { + var fields = new[] + { + new TextFieldSelector + { + Name = $"deviceProperties.{key}", + Boost = 10, + Type = TextFieldType.Exact, + }, + new TextFieldSelector + { + Name = $"deviceProperties.{key}", + Boost = 8, + Type = TextFieldType.PrefixWildcard, + }, + }; + + propFilters.Add(mf + => mf.Nested(nqd + => nqd + .Path(d => d.DeviceProperties) + .Query(q => Util.ToTextQuery(q, value, fields)))); + } + } + return sd => sd .MinScore(mustFilters.Any() ? 8 : 0) .Query(qd => qd.Bool(bqd => - { - var shouldQueries = request.InterfaceIds.Select(ifaceId - => (Func, QueryContainer>) (sf - => sf.Match(f => f.Field(str => str.InterfaceId).Query($"{ifaceId}")))) - .ToArray(); + { + var shouldQueries = request.InterfaceIds.Select(ifaceId + => (Func, QueryContainer>) (sf + => sf.Match(f => f.Field(str => str.InterfaceId).Query($"{ifaceId}")))) + .ToArray(); - return bqd.Should(shouldQueries).Must(mustFilters.ToArray()); - })); + return bqd.Should(shouldQueries).Must(mustFilters.ToArray()).Should(propFilters); + })); } + + private static Func, SearchDescriptor> FilterByInterfaceIds( + IEnumerable ifaceIds) + => sd + => sd.Query(qd + => qd.Bool(bqd => bqd.Must(mf + => mf.Terms(f + => f.Field(str => str.InterfaceId).Terms(ifaceIds.Select(id => $"{id}")))))); }