diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01ddaba25..f61a58554 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,8 @@ on: - 'master' - 'dev' - 'release/*' + - 'support/*' + - 'hotfix/*' tags: - 'v*' pull_request: @@ -14,6 +16,8 @@ on: - 'master' - 'dev' - 'release/*' + - 'support/*' + - 'hotfix/*' jobs: build: @@ -40,18 +44,17 @@ jobs: - name: Setup dotnet uses: actions/setup-dotnet@v3 with: - dotnet-version: | - 5.0.x + dotnet-version: | 6.0.x - 7.0.x + 8.0.x - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v0.9.9 + uses: gittools/actions/gitversion/setup@v3.0.3 with: - versionSpec: '5.x' + versionSpec: '6.0.x' - name: Determine Version - uses: gittools/actions/gitversion/execute@v0.9.9 + uses: gittools/actions/gitversion/execute@v3.0.3 - name: Install dependencies run: dotnet restore ${{ env.Solution_File }} @@ -60,10 +63,10 @@ jobs: run: dotnet build ${{ env.Solution_File }} --configuration ${{ env.Configuration }} --no-restore - name: Test - run: dotnet test "${{ env.Test_Proj }}" --no-build --verbosity normal --results-directory ${{ github.workspace }}/_TestResults --logger "trx;logfilename=tests.trx" + run: dotnet test "${{ env.Test_Proj }}" --no-build --verbosity normal --results-directory ${{ github.workspace }}/_TestResults --logger "trx;logfilename=tests.trx" --filter "TestCategory!~Benchmarks" - name: Upload test results - uses: actions/upload-artifact@v2 # upload test results + uses: actions/upload-artifact@v4 # upload test results if: success() || failure() # run this step even if previous step failed with: name: examine-test-results @@ -85,7 +88,7 @@ jobs: --output=${{ github.workspace }}/_NugetOutput - name: Upload artifacts - uses: actions/upload-artifact@v2 # upload nuget + uses: actions/upload-artifact@v4 # upload nuget if: success() with: name: examine-nuget-${{ env.GitVersion_SemVer }} diff --git a/.github/workflows/docfx-gh-pages.yml b/.github/workflows/docfx-gh-pages.yml index 9ec09b6d9..e6adb7e4f 100644 --- a/.github/workflows/docfx-gh-pages.yml +++ b/.github/workflows/docfx-gh-pages.yml @@ -4,6 +4,9 @@ on: # Runs on pushes targeting the default branch push: branches: ["feature/docfx"] + pull_request: + branches: + - 'feature/docfx' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml index a6efea569..708b72604 100644 --- a/.github/workflows/test-report.yml +++ b/.github/workflows/test-report.yml @@ -8,7 +8,7 @@ jobs: report: runs-on: ubuntu-latest steps: - - uses: dorny/test-reporter@v1.4.2 + - uses: dorny/test-reporter@v1 with: artifact: examine-test-results # artifact name name: Publish Tests # Name of the check run which will be created diff --git a/GitVersion.yml b/GitVersion.yml.bak similarity index 68% rename from GitVersion.yml rename to GitVersion.yml.bak index 54aa6012e..e0a053a84 100644 --- a/GitVersion.yml +++ b/GitVersion.yml.bak @@ -17,16 +17,5 @@ commit-message-incrementing: Enabled commit-date-format: 'yyyy-MM-dd' ignore: sha: [] - commits-before: 2020-12-21T00:00:00 -merge-message-formats: {} -branches: - main: - regex: ^master(\-0\.x)?$|^main(\-0\.x)?$ - source-branches: [] - develop: - regex: ^dev(elop)?(ment)?(\-0\.x)?$ - source-branches: - - main - release: - mode: ContinuousDeployment - source-branches: ['develop'] \ No newline at end of file + commits-before: 2021-01-01T00:00:00 +merge-message-formats: {} \ No newline at end of file diff --git a/docs/articles/indexing.md b/docs/articles/indexing.md index 7ad7ca846..bf7b539d4 100644 --- a/docs/articles/indexing.md +++ b/docs/articles/indexing.md @@ -115,9 +115,10 @@ myIndex.IndexItem(new ValueSet( Be default all indexing is done asynchronously. If you need to run indexing synchronously you should create a synchronous scope. This is for instance a necessary step for unit tests. ```cs -using (myIndex.ProcessNonAsync()) +using (var luceneIndex = (LuceneIndex)myIndex) +using (var syncIndexContext = luceneIndex.WithThreadingMode(IndexThreadingMode.Synchronous)) { - myIndex.IndexItem(new ValueSet( + luceneIndex.IndexItem(new ValueSet( "SKU987", "Product", new Dictionary() @@ -251,4 +252,4 @@ private void IndexCommited(object sender, EventArgs e) { // Triggered when the index is commited } -``` \ No newline at end of file +``` diff --git a/docs/articles/searching.md b/docs/articles/searching.md index 6b2fd0250..4a454ee9a 100644 --- a/docs/articles/searching.md +++ b/docs/articles/searching.md @@ -132,7 +132,7 @@ query.Field("nodeTypeAlias", "CWS_Home".Boost(20)); var results = query.Execute(); ``` -This will boost the term `CWS_Home` and make enteries with `nodeTypeAlias:CWS_Home` score higher in the results. +This will boost the term `CWS_Home` and make entries with `nodeTypeAlias:CWS_Home` score higher in the results. ## Proximity diff --git a/docs/docs-v1-v2/searching.md b/docs/docs-v1-v2/searching.md index 2905d7804..6a2941580 100644 --- a/docs/docs-v1-v2/searching.md +++ b/docs/docs-v1-v2/searching.md @@ -124,7 +124,7 @@ query.Field("nodeTypeAlias", "CWS_Home".Boost(20)); var results = query.Execute(); ``` -This will boost the term `CWS_Home` and make enteries with `nodeTypeAlias:CWS_Home` score higher in the results. +This will boost the term `CWS_Home` and make entries with `nodeTypeAlias:CWS_Home` score higher in the results. ## Proximity @@ -248,4 +248,4 @@ var query = searcher.CreateQuery(); var query = (LuceneSearchQuery)query.NativeQuery("hello:world").And(); // Make query ready for extending query.LuceneQuery(NumericRangeQuery.NewInt64Range("numTest", 4, 5, true, true)); // Add the raw lucene query var results = query.Execute(); -``` \ No newline at end of file +``` diff --git a/src/Directory.Build.props b/src/Directory.Build.props index ade8d2262..206a89fdc 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -6,11 +6,16 @@ true true snupkg + true + NU5104 - + + + true + https://github.com/Shazwazza/Examine https://github.com/Shazwazza/Examine git @@ -19,14 +24,12 @@ shandem logo-round-small.png MS-PL - 2.0.0 - - - net6.0;netstandard2.1;netstandard2.0 + 4.0.0 - - - $(NoWarn);nullable + + net6.0;net8.0; + latest + enable diff --git a/src/Examine.Benchmarks/ConcurrentAcquireBenchmarks.cs b/src/Examine.Benchmarks/ConcurrentAcquireBenchmarks.cs new file mode 100644 index 000000000..f3f0f03c2 --- /dev/null +++ b/src/Examine.Benchmarks/ConcurrentAcquireBenchmarks.cs @@ -0,0 +1,143 @@ +using System.Diagnostics; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Jobs; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Lucene.Net.Util; +using Microsoft.Extensions.Logging; + +namespace Examine.Benchmarks +{ + [MediumRunJob(RuntimeMoniker.Net80)] + [ThreadingDiagnoser] + [MemoryDiagnoser] + public class ConcurrentAcquireBenchmarks : ExamineBaseTest + { + private readonly StandardAnalyzer _analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + private string? _tempBasePath; + private FSDirectory? _indexDir; + private IndexWriter? _writer; + private SearcherManager? _searcherManager; + + [GlobalSetup] + public override void Setup() + { + base.Setup(); + + _tempBasePath = Path.Combine(Path.GetTempPath(), "ExamineTests"); + + // indexer for lucene + InitializeAndIndexItems(_tempBasePath, _analyzer, out var indexDir); + _indexDir = FSDirectory.Open(indexDir); + var writerConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, _analyzer); + //writerConfig.SetMaxBufferedDocs(1000); + //writerConfig.SetReaderTermsIndexDivisor(4); + //writerConfig.SetOpenMode(OpenMode.APPEND); + //writerConfig.SetReaderPooling(true); + //writerConfig.SetCodec(new Lucene46Codec()); + _writer = new IndexWriter(_indexDir, writerConfig); + var trackingWriter = new TrackingIndexWriter(_writer); + _searcherManager = new SearcherManager(trackingWriter.IndexWriter, applyAllDeletes: true, new SearcherFactory()); + } + + [GlobalCleanup] + public override void TearDown() + { + _searcherManager?.Dispose(); + _writer?.Dispose(); + _indexDir?.Dispose(); + + base.TearDown(); + + System.IO.Directory.Delete(_tempBasePath!, true); + } + + [Params(1, 15, 30, 100)] + public int ThreadCount { get; set; } + + [Benchmark(Baseline = true)] + public async Task SimpleMultiThreadLoop() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => Debug.Write(i))); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + + [Benchmark] + public async Task TestAcquireThreadContention() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + var searcher = _searcherManager!.Acquire(); + try + { + if (searcher.IndexReader.RefCount > (ThreadCount + 1)) + { + Console.WriteLine(searcher.IndexReader.RefCount); + } + } + finally + { + _searcherManager.Release(searcher); + } + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + +#if RELEASE + protected override ILoggerFactory CreateLoggerFactory() + => Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Information)); +#endif + private void InitializeAndIndexItems( + string tempBasePath, + Analyzer analyzer, + out DirectoryInfo indexDir) + { + var tempPath = Path.Combine(tempBasePath, Guid.NewGuid().ToString()); + System.IO.Directory.CreateDirectory(tempPath); + indexDir = new DirectoryInfo(tempPath); + using var luceneDirectory = FSDirectory.Open(indexDir); + using var luceneTaxonomyDir = FSDirectory.Open(Path.Combine(tempPath, "Taxonomy")); + using var indexer = GetTestIndex(luceneDirectory, luceneTaxonomyDir, analyzer); + + var random = new Random(); + var valueSets = new List(); + + for (var i = 0; i < 1000; i++) + { + valueSets.Add(ValueSet.FromObject(Guid.NewGuid().ToString(), "content", + new + { + nodeName = "location " + i, + bodyText = Enumerable.Range(0, random.Next(10, 100)).Select(x => Guid.NewGuid().ToString()) + })); + } + + indexer.IndexItems(valueSets); + } + } +} diff --git a/src/Examine.Benchmarks/ConcurrentSearchBenchmarks.cs b/src/Examine.Benchmarks/ConcurrentSearchBenchmarks.cs new file mode 100644 index 000000000..d458b7366 --- /dev/null +++ b/src/Examine.Benchmarks/ConcurrentSearchBenchmarks.cs @@ -0,0 +1,555 @@ +#if LocalBuild +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Jobs; +using Examine.Lucene.Search; +using Examine.Search; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Index; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Lucene.Net.Util; +using Microsoft.Extensions.Logging; + +//[assembly: Config(typeof(MyDefaultConfig))] + +//internal class MyDefaultConfig : ManualConfig +//{ +// public MyDefaultConfig() +// { +// WithOptions(ConfigOptions.DisableOptimizationsValidator); +// } +//} + +namespace Examine.Benchmarks +{ + /* + + Original + + | Method | ThreadCount | Mean | Error | StdDev | Completed Work Items | Lock Contentions | Gen0 | Gen1 | Gen2 | Allocated | + |---------------- |------------ |-----------:|------------:|-----------:|---------------------:|-----------------:|----------:|----------:|----------:|----------:| + |---------------- |------------ |-----------:|------------:|-----------:|---------------------:|-----------------:|----------:|----------:|----------:|----------:| + | ExamineStandard | 1 | 8.712 ms | 0.6798 ms | 0.0373 ms | 1.0000 | - | 234.3750 | 140.6250 | - | 2.86 MB | + | LuceneSimple | 1 | 9.723 ms | 0.4864 ms | 0.0267 ms | 1.0000 | 0.0469 | 250.0000 | 234.3750 | - | 3.01 MB | + | ExamineStandard | 5 | 154.451 ms | 39.5553 ms | 2.1682 ms | 5.0000 | - | 1000.0000 | 750.0000 | - | 14.3 MB | + | LuceneSimple | 5 | 16.953 ms | 6.1768 ms | 0.3386 ms | 5.0000 | - | 1250.0000 | 1000.0000 | 93.7500 | 15.06 MB | + | ExamineStandard | 15 | 657.503 ms | 195.5415 ms | 10.7183 ms | 15.0000 | - | 3000.0000 | 1000.0000 | - | 42.92 MB | + | LuceneSimple | 15 | 60.278 ms | 100.6474 ms | 5.5168 ms | 15.0000 | - | 4333.3333 | 2666.6667 | 1000.0000 | 45.2 MB | + + Without NRT (no diff really) + + | Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | + |---------------- |------------ |----------:|-----------:|----------:|----------:|---------------------:|-----------------:|----------:|----------:| + | ExamineStandard | 1 | 12.48 ms | 3.218 ms | 0.176 ms | 250.0000 | 1.0000 | 0.0938 | 156.2500 | 3.13 MB | + | ExamineStandard | 5 | 149.31 ms | 88.914 ms | 4.874 ms | 1000.0000 | 5.0000 | 4.0000 | 750.0000 | 14.7 MB | + | ExamineStandard | 15 | 613.14 ms | 897.936 ms | 49.219 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.67 MB | + + Without querying MaxDoc (Shows we were double/triple querying) + + | Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | + |---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| + | ExamineStandard | 1 | 5.223 ms | 1.452 ms | 0.0796 ms | 78.1250 | 1.0000 | 0.0313 | 7.8125 | 962 KB | + | ExamineStandard | 5 | 26.772 ms | 9.982 ms | 0.5471 ms | 312.5000 | 5.0000 | 4.0000 | 187.5000 | 3825.35 KB | + | ExamineStandard | 15 | 101.483 ms | 65.690 ms | 3.6007 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10989.05 KB | + + Without apply deletes (should be faster, we'll keep it) + UPDATE: We cannot, that is specialized and we cannot support it. + + | Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | + |---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| + | ExamineStandard | 1 | 5.554 ms | 1.745 ms | 0.0957 ms | 78.1250 | 1.0000 | - | 31.2500 | 961.73 KB | + | ExamineStandard | 5 | 26.960 ms | 4.797 ms | 0.2629 ms | 312.5000 | 5.0000 | 4.0313 | 187.5000 | 3826.6 KB | + | ExamineStandard | 15 | 103.939 ms | 49.361 ms | 2.7057 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10991.87 KB | + + Using struct (doesn't change anything) + + | Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | + |---------------- |------------ |-----------:|----------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| + | ExamineStandard | 1 | 5.661 ms | 2.477 ms | 0.1357 ms | 78.1250 | 1.0000 | 0.0625 | 31.2500 | 961.56 KB | + | ExamineStandard | 5 | 28.364 ms | 3.615 ms | 0.1981 ms | 312.5000 | 5.0000 | 4.0000 | 187.5000 | 3825.91 KB | + | ExamineStandard | 15 | 100.561 ms | 26.820 ms | 1.4701 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10986.15 KB | + + With Latest changes (don't re-create SearchContext, cache fields if nothing changes, etc...): + + | Method | ThreadCount | Mean | Error | StdDev | Completed Work Items | Lock Contentions | Gen0 | Gen1 | Allocated | + |---------------- |------------ |-----------:|------------:|-----------:|---------------------:|-----------------:|----------:|----------:|------------:| + | ExamineStandard | 1 | 5.157 ms | 1.0374 ms | 0.0569 ms | 1.0000 | 0.0156 | 78.1250 | 39.0625 | 963.3 KB | + | LuceneSimple | 1 | 11.338 ms | 0.8416 ms | 0.0461 ms | 1.0000 | 0.0156 | 265.6250 | 187.5000 | 3269.09 KB | + | ExamineStandard | 5 | 27.038 ms | 7.2847 ms | 0.3993 ms | 5.0000 | 4.0000 | 312.5000 | 187.5000 | 3812.7 KB | + | LuceneSimple | 5 | 144.196 ms | 185.2203 ms | 10.1526 ms | 5.0000 | - | 1000.0000 | 750.0000 | 15047.06 KB | + | ExamineStandard | 15 | 95.799 ms | 64.1371 ms | 3.5156 ms | 15.0000 | 14.0000 | 833.3333 | 500.0000 | 10940.31 KB | + | LuceneSimple | 15 | 566.652 ms | 275.2278 ms | 15.0862 ms | 15.0000 | - | 3000.0000 | 1000.0000 | 44485.6 KB | + + Determining the best NRT values + + | Method | ThreadCount | NrtTargetMaxStaleSec | NrtTargetMinStaleSec | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | + |---------------- |------------ |--------------------- |--------------------- |-----------:|------------:|----------:|---------:|---------------------:|-----------------:|---------:|------------:| + | ExamineStandard | 1 | 5 | 1 | 5.507 ms | 1.7993 ms | 0.0986 ms | 78.1250 | 1.0000 | - | 31.2500 | 963.59 KB | + | ExamineStandard | 1 | 5 | 5 | 5.190 ms | 0.4792 ms | 0.0263 ms | 78.1250 | 1.0000 | 0.0078 | 39.0625 | 963.65 KB | + | ExamineStandard | 1 | 60 | 1 | 5.406 ms | 2.2636 ms | 0.1241 ms | 78.1250 | 1.0000 | 0.0313 | 31.2500 | 963.71 KB | + | ExamineStandard | 1 | 60 | 5 | 5.316 ms | 3.4301 ms | 0.1880 ms | 78.1250 | 1.0000 | - | 39.0625 | 963.42 KB | + | ExamineStandard | 5 | 5 | 1 | 26.439 ms | 1.2601 ms | 0.0691 ms | 312.5000 | 5.0000 | 4.0000 | 187.5000 | 3813.45 KB | + | ExamineStandard | 5 | 5 | 5 | 27.341 ms | 13.3950 ms | 0.7342 ms | 312.5000 | 5.0000 | 4.0313 | 187.5000 | 3813.83 KB | + | ExamineStandard | 5 | 60 | 1 | 26.768 ms | 9.4732 ms | 0.5193 ms | 312.5000 | 5.0000 | 4.0000 | 156.2500 | 3814.06 KB | + | ExamineStandard | 5 | 60 | 5 | 27.216 ms | 3.3213 ms | 0.1821 ms | 312.5000 | 5.0000 | 4.0000 | 187.5000 | 3813.83 KB | + | ExamineStandard | 15 | 5 | 1 | 101.040 ms | 44.3254 ms | 2.4296 ms | 800.0000 | 15.0000 | 14.0000 | 600.0000 | 10940.73 KB | + | ExamineStandard | 15 | 5 | 5 | 104.027 ms | 44.7547 ms | 2.4532 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10939.87 KB | + | ExamineStandard | 15 | 60 | 1 | 96.622 ms | 162.1682 ms | 8.8890 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10941.64 KB | + | ExamineStandard | 15 | 60 | 5 | 102.469 ms | 78.0316 ms | 4.2772 ms | 800.0000 | 15.0000 | 14.0000 | 400.0000 | 10936.86 KB | + + Putting MaxDoc back in makes it go crazy + + | Method | ThreadCount | Mean | Error | StdDev | Gen0 | Completed Work Items | Lock Contentions | Gen1 | Allocated | + |---------------- |------------ |----------:|-----------:|----------:|----------:|---------------------:|-----------------:|----------:|----------:| + | ExamineStandard | 1 | 12.90 ms | 4.049 ms | 0.222 ms | 250.0000 | 1.0000 | - | 156.2500 | 3.13 MB | + | ExamineStandard | 5 | 149.16 ms | 74.884 ms | 4.105 ms | 1000.0000 | 5.0000 | 4.0000 | 750.0000 | 14.69 MB | + | ExamineStandard | 15 | 635.77 ms | 899.620 ms | 49.311 ms | 3000.0000 | 15.0000 | 14.0000 | 1000.0000 | 43.57 MB | + + Using different MaxResults leads to crazy results + + | Method | ThreadCount | MaxResults | Mean | Error | StdDev | Completed Work Items | Lock Contentions | Gen0 | Gen1 | Gen2 | Allocated | + |---------------- |------------ |----------- |-------------:|--------------:|------------:|---------------------:|-----------------:|----------:|----------:|----------:|----------:| + | ExamineStandard | 15 | 10 | 4.979 ms | 1.6928 ms | 0.0928 ms | 15.0000 | 14.0000 | 257.8125 | 109.3750 | - | 3 MB | + | LuceneSimple | 15 | 10 | 4.168 ms | 0.6606 ms | 0.0362 ms | 15.0000 | 0.0234 | 218.7500 | 93.7500 | - | 2.57 MB | + | ExamineStandard | 15 | 100 | 92.838 ms | 88.3517 ms | 4.8429 ms | 15.0000 | 14.0000 | 833.3333 | 666.6667 | - | 10.68 MB | + | LuceneSimple | 15 | 100 | 103.927 ms | 64.1171 ms | 3.5145 ms | 15.0000 | - | 800.0000 | 600.0000 | - | 10.33 MB | + | ExamineStandard | 15 | 1000 | 1,278.769 ms | 826.1505 ms | 45.2841 ms | 15.0000 | 14.0000 | 7000.0000 | 4000.0000 | 1000.0000 | 84.55 MB | + | LuceneSimple | 15 | 1000 | 1,248.199 ms | 1,921.5844 ms | 105.3285 ms | 15.0000 | - | 7000.0000 | 4000.0000 | 1000.0000 | 84.08 MB | + + After changing to use singleton indexers/managers + + | Method | ThreadCount | MaxResults | Mean | Error | StdDev | Completed Work Items | Lock Contentions | Gen0 | Gen1 | Gen2 | Allocated | + |---------------- |------------ |----------- |---------------:|--------------:|-------------:|---------------------:|-----------------:|-----------:|-----------:|----------:|-------------:| + | ExamineStandard | 1 | 10 | 101.9 μs | 9.70 μs | 0.53 μs | 1.0000 | 0.0029 | 12.6953 | 0.9766 | - | 157.77 KB | + | LuceneSimple | 1 | 10 | 120.7 us | 9.33 us | 0.51 us | 1.0000 | 0.0022 | 11.4746 | 1.2207 | - | 141.66 KB | + | ExamineStandard | 1 | 100 | 1,555.0 us | 407.07 us | 22.31 us | 1.0000 | 0.0078 | 54.6875 | 15.6250 | - | 681.92 KB | + | LuceneSimple | 1 | 100 | 1,598.8 μs | 233.79 μs | 12.81 μs | 1.0000 | 0.0078 | 52.7344 | 17.5781 | - | 664.64 KB | + | ExamineStandard | 1 | 1000 | 17,449.3 μs | 1,472.32 μs | 80.70 μs | 1.0000 | - | 437.5000 | 312.5000 | 31.2500 | 5723.12 KB | + | LuceneSimple | 1 | 1000 | 17,739.7 μs | 3,797.03 μs | 208.13 μs | 1.0000 | 0.0313 | 437.5000 | 312.5000 | 31.2500 | 5698.42 KB | + | ExamineStandard | 15 | 10 | 1,630.6 μs | 2,436.46 μs | 133.55 μs | 15.0000 | 0.0430 | 195.3125 | 15.6250 | - | 2362.51 KB | + | LuceneSimple | 15 | 10 | 1,742.6 μs | 214.81 μs | 11.77 μs | 15.0000 | 0.0820 | 179.6875 | 27.3438 | - | 2118.47 KB | + | ExamineStandard | 15 | 100 | 105,817.2 μs | 28,398.55 μs | 1,556.62 μs | 15.0000 | - | 833.3333 | 666.6667 | - | 10225.39 KB | + | LuceneSimple | 15 | 100 | 95,732.1 μs | 57,903.39 μs | 3,173.88 μs | 15.0000 | - | 666.6667 | 500.0000 | - | 9967.2 KB | + | ExamineStandard | 15 | 1000 | 1,125,955.0 μs | 822,782.38 μs | 45,099.48 μs | 15.0000 | - | 7000.0000 | 4000.0000 | 1000.0000 | 85877.8 KB | + | LuceneSimple | 15 | 1000 | 1,446,507.5 μs | 855,107.53 μs | 46,871.33 μs | 15.0000 | - | 7000.0000 | 4000.0000 | 1000.0000 | 85509.77 KB | + | ExamineStandard | 30 | 10 | 4,261.3 μs | 1,676.61 μs | 91.90 μs | 30.0000 | 0.3047 | 390.6250 | 70.3125 | - | 4724.59 KB | + | LuceneSimple | 30 | 10 | 3,895.8 μs | 1,768.88 μs | 96.96 μs | 30.0000 | 0.1250 | 359.3750 | 46.8750 | - | 4237.24 KB | + | ExamineStandard | 30 | 100 | 232,909.0 μs | 30,215.14 μs | 1,656.19 μs | 30.0000 | - | 1500.0000 | 1000.0000 | - | 20455.26 KB | + | LuceneSimple | 30 | 100 | 259,557.3 μs | 40,643.51 μs | 2,227.81 μs | 30.0000 | - | 1500.0000 | 1000.0000 | - | 19940.39 KB | + | ExamineStandard | 30 | 1000 | 2,886,589.2 μs | 328,362.02 μs | 17,998.63 μs | 30.0000 | 1.0000 | 16000.0000 | 11000.0000 | 3000.0000 | 171858.03 KB | + | LuceneSimple | 30 | 1000 | 2,662,715.9 μs | 898,686.63 μs | 49,260.05 μs | 30.0000 | - | 16000.0000 | 11000.0000 | 3000.0000 | 171094.02 KB | + + + */ + [LongRunJob(RuntimeMoniker.Net80)] + [ThreadingDiagnoser] + [MemoryDiagnoser] + //[DotNetCountersDiagnoser] + //[CPUUsageDiagnoser] + public class ConcurrentSearchBenchmarks : ExamineBaseTest + { + private readonly StandardAnalyzer _analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + private ILogger? _logger; + private string? _tempBasePath; + private TestIndex? _indexer; + private FSDirectory? _indexDir; + private IndexWriter? _writer; + private SearcherManager? _searcherManager; + + [GlobalSetup] + public override void Setup() + { + base.Setup(); + + _logger = LoggerFactory!.CreateLogger(); + _tempBasePath = Path.Combine(Path.GetTempPath(), "ExamineTests"); + + // indexer for examine + _indexer = InitializeAndIndexItems(_tempBasePath, _analyzer, out _); + + // indexer for lucene + var tempIndexer = InitializeAndIndexItems(_tempBasePath, _analyzer, out var indexDir); + tempIndexer.Dispose(); + _indexDir = FSDirectory.Open(indexDir); + var writerConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, _analyzer); + //writerConfig.SetMaxBufferedDocs(1000); + //writerConfig.SetReaderTermsIndexDivisor(4); + //writerConfig.SetOpenMode(OpenMode.APPEND); + //writerConfig.SetReaderPooling(true); + //writerConfig.SetCodec(new Lucene46Codec()); + _writer = new IndexWriter(_indexDir, writerConfig); + var trackingWriter = new TrackingIndexWriter(_writer); + _searcherManager = new SearcherManager(trackingWriter.IndexWriter, applyAllDeletes: true, new SearcherFactory()); + } + + [GlobalCleanup] + public override void TearDown() + { + _indexer?.Dispose(); + _searcherManager?.Dispose(); + _writer?.Dispose(); + _indexDir?.Dispose(); + + base.TearDown(); + + System.IO.Directory.Delete(_tempBasePath!, true); + } + + [Params(1, 50, 100)] + public int ThreadCount { get; set; } + + [Params(10/*, 100, 1000*/)] + public int MaxResults { get; set; } + + [Benchmark(Baseline = true)] + public async Task ExamineStandard() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + // always resolve the searcher from the indexer + var searcher = _indexer!.Searcher; + + var query = searcher.CreateQuery("content").Field("nodeName", "location".MultipleCharacterWildcard()); + var results = query.Execute(QueryOptions.SkipTake(0, MaxResults)); + + // enumerate (forces the result to execute) + var logOutput = "ThreadID: " + Thread.CurrentThread.ManagedThreadId + ", Results: " + string.Join(',', results.Select(x => $"{x.Id}-{x.Values.Count}-{x.Score}").ToArray()); + _logger!.LogDebug(logOutput); + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + + [Benchmark] + public async Task SimpleMultiThreadLoop() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + + public async Task TestAcquireThreadContention() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + var parser = new QueryParser(LuceneVersion.LUCENE_48, ExamineFieldNames.ItemIdFieldName, new StandardAnalyzer(LuceneVersion.LUCENE_48)); + var query = parser.Parse($"{ExamineFieldNames.CategoryFieldName}:content AND nodeName:location*"); + + // this is like doing Acquire, does it perform the same (it will allocate more) + using var context = _searcherManager.GetContext(); + + var searcher = context.Reference; + + // Don't use this, increasing the max docs substantially decreases performance + //var maxDoc = searcher.IndexReader.MaxDoc; + var topDocsCollector = TopScoreDocCollector.Create(MaxResults, null, true); + + searcher.Search(query, topDocsCollector); + + var topDocs = topDocsCollector.GetTopDocs(0, MaxResults); + + var totalItemCount = topDocs.TotalHits; + var maxScore = topDocs.MaxScore; + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + + [Benchmark] + public async Task LuceneAcquireAlways() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + var parser = new QueryParser(LuceneVersion.LUCENE_48, ExamineFieldNames.ItemIdFieldName, new StandardAnalyzer(LuceneVersion.LUCENE_48)); + var query = parser.Parse($"{ExamineFieldNames.CategoryFieldName}:content AND nodeName:location*"); + + // this is like doing Acquire, does it perform the same (it will allocate more) + using var context = _searcherManager.GetContext(); + + var searcher = context.Reference; + + // Don't use this, increasing the max docs substantially decreases performance + //var maxDoc = searcher.IndexReader.MaxDoc; + var topDocsCollector = TopScoreDocCollector.Create(MaxResults, null, true); + + searcher.Search(query, topDocsCollector); + + var topDocs = topDocsCollector.GetTopDocs(0, MaxResults); + + var totalItemCount = topDocs.TotalHits; + + var results = new List(topDocs.ScoreDocs.Length); + + foreach (var scoreDoc in topDocs.ScoreDocs) + { + var docId = scoreDoc.Doc; + var score = scoreDoc.Score; + var shardIndex = scoreDoc.ShardIndex; + var doc = searcher.Doc(docId); + var result = LuceneSearchExecutor.CreateSearchResult(doc, score, shardIndex); + results.Add(result); + } + + var maxScore = topDocs.MaxScore; + + // enumerate (forces the result to execute) + var logOutput = "ThreadID: " + Thread.CurrentThread.ManagedThreadId + ", Results: " + string.Join(',', results.Select(x => $"{x.Id}-{x.Values.Count}-{x.Score}").ToArray()); + _logger!.LogDebug(logOutput); + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + + [Benchmark] + public async Task LuceneAcquireAlwaysWithLock() + { + var tasks = new List(); + var myLock = new object(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + lock (myLock) + { + var parser = new QueryParser(LuceneVersion.LUCENE_48, ExamineFieldNames.ItemIdFieldName, new StandardAnalyzer(LuceneVersion.LUCENE_48)); + var query = parser.Parse($"{ExamineFieldNames.CategoryFieldName}:content AND nodeName:location*"); + + // this is like doing Acquire, does it perform the same (it will allocate more) + using var context = _searcherManager.GetContext(); + + var searcher = context.Reference; + + // Don't use this, increasing the max docs substantially decreases performance + //var maxDoc = searcher.IndexReader.MaxDoc; + var topDocsCollector = TopScoreDocCollector.Create(MaxResults, null, true); + + searcher.Search(query, topDocsCollector); + + var topDocs = topDocsCollector.GetTopDocs(0, MaxResults); + + var totalItemCount = topDocs.TotalHits; + + var results = new List(topDocs.ScoreDocs.Length); + + foreach (var scoreDoc in topDocs.ScoreDocs) + { + var docId = scoreDoc.Doc; + var score = scoreDoc.Score; + var shardIndex = scoreDoc.ShardIndex; + var doc = searcher.Doc(docId); + var result = LuceneSearchExecutor.CreateSearchResult(doc, score, shardIndex); + results.Add(result); + } + + var maxScore = topDocs.MaxScore; + + // enumerate (forces the result to execute) + var logOutput = "ThreadID: " + Thread.CurrentThread.ManagedThreadId + ", Results: " + string.Join(',', results.Select(x => $"{x.Id}-{x.Values.Count}-{x.Score}").ToArray()); + _logger!.LogDebug(logOutput); + } + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + + [Benchmark] + public async Task LuceneAcquireOnce() + { + var tasks = new List(); + + var searcher = _searcherManager!.Acquire(); + + try + { + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + var parser = new QueryParser(LuceneVersion.LUCENE_48, ExamineFieldNames.ItemIdFieldName, new StandardAnalyzer(LuceneVersion.LUCENE_48)); + var query = parser.Parse($"{ExamineFieldNames.CategoryFieldName}:content AND nodeName:location*"); + + // Don't use this, increasing the max docs substantially decreases performance + //var maxDoc = searcher.IndexReader.MaxDoc; + var topDocsCollector = TopScoreDocCollector.Create(MaxResults, null, true); + + searcher.Search(query, topDocsCollector); + var topDocs = topDocsCollector.GetTopDocs(0, MaxResults); + + var totalItemCount = topDocs.TotalHits; + + var results = new List(topDocs.ScoreDocs.Length); + for (var i = 0; i < topDocs.ScoreDocs.Length; i++) + { + var scoreDoc = topDocs.ScoreDocs[i]; + var docId = scoreDoc.Doc; + var doc = searcher.Doc(docId); + var score = scoreDoc.Score; + var shardIndex = scoreDoc.ShardIndex; + var result = LuceneSearchExecutor.CreateSearchResult(doc, score, shardIndex); + results.Add(result); + } + + var maxScore = topDocs.MaxScore; + + // enumerate (forces the result to execute) + var logOutput = "ThreadID: " + Thread.CurrentThread.ManagedThreadId + ", Results: " + string.Join(',', results.Select(x => $"{x.Id}-{x.Values.Count}-{x.Score}").ToArray()); + _logger!.LogDebug(logOutput); + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + finally + { + _searcherManager.Release(searcher); + } + } + + [Benchmark] + public async Task LuceneSortedDocIds() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + var parser = new QueryParser(LuceneVersion.LUCENE_48, ExamineFieldNames.ItemIdFieldName, new StandardAnalyzer(LuceneVersion.LUCENE_48)); + var query = parser.Parse($"{ExamineFieldNames.CategoryFieldName}:content AND nodeName:location*"); + + // this is like doing Acquire, does it perform the same (it will allocate more) + using var context = _searcherManager.GetContext(); + + var searcher = context.Reference; + + // Don't use this, increasing the max docs substantially decreases performance + //var maxDoc = searcher.IndexReader.MaxDoc; + var topDocsCollector = TopScoreDocCollector.Create(MaxResults, null, true); + + searcher.Search(query, topDocsCollector); + + var topDocs = topDocsCollector.GetTopDocs(0, MaxResults); + + var totalItemCount = topDocs.TotalHits; + + var results = new List(topDocs.ScoreDocs.Length); + + foreach (var scoreDoc in topDocs.ScoreDocs.OrderBy(x => x.Doc)) + { + var docId = scoreDoc.Doc; + var score = scoreDoc.Score; + var shardIndex = scoreDoc.ShardIndex; + var doc = searcher.Doc(docId); + var result = LuceneSearchExecutor.CreateSearchResult(doc, score, shardIndex); + results.Add(result); + } + + var maxScore = topDocs.MaxScore; + + // enumerate (forces the result to execute) + var logOutput = "ThreadID: " + Thread.CurrentThread.ManagedThreadId + ", Results: " + string.Join(',', results.Select(x => $"{x.Id}-{x.Values.Count}-{x.Score}").ToArray()); + _logger!.LogDebug(logOutput); + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + +#if RELEASE + protected override ILoggerFactory CreateLoggerFactory() + => Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Information)); +#endif + private TestIndex InitializeAndIndexItems( + string tempBasePath, + Analyzer analyzer, + out DirectoryInfo indexDir) + { + var tempPath = Path.Combine(tempBasePath, Guid.NewGuid().ToString()); + System.IO.Directory.CreateDirectory(tempPath); + indexDir = new DirectoryInfo(tempPath); + var luceneDirectory = FSDirectory.Open(indexDir); + var luceneTaxonomyDir = FSDirectory.Open(Path.Combine(tempPath, "Taxonomy")); + var indexer = GetTestIndex(luceneDirectory, luceneTaxonomyDir, analyzer); + + var random = new Random(); + var valueSets = new List(); + + for (var i = 0; i < 1000; i++) + { + valueSets.Add(ValueSet.FromObject(Guid.NewGuid().ToString(), "content", + new + { + nodeName = "location " + i, + bodyText = Enumerable.Range(0, random.Next(10, 100)).Select(x => Guid.NewGuid().ToString()) + })); + } + + indexer.IndexItems(valueSets); + + return indexer; + } + } +} + +#endif diff --git a/src/Examine.Benchmarks/Examine.Benchmarks.csproj b/src/Examine.Benchmarks/Examine.Benchmarks.csproj new file mode 100644 index 000000000..c51553ea6 --- /dev/null +++ b/src/Examine.Benchmarks/Examine.Benchmarks.csproj @@ -0,0 +1,29 @@ + + + + net8.0 + enable + enable + false + false + Exe + + true + $(DefineConstants);LocalBuild + + + + + + + + + + + + + + + + + diff --git a/src/Examine.Benchmarks/ExamineBaseTest.cs b/src/Examine.Benchmarks/ExamineBaseTest.cs new file mode 100644 index 000000000..2afff05fe --- /dev/null +++ b/src/Examine.Benchmarks/ExamineBaseTest.cs @@ -0,0 +1,66 @@ +using Examine.Lucene; +using Examine.Lucene.Directories; +using Lucene.Net.Analysis; +using Lucene.Net.Index; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Directory = Lucene.Net.Store.Directory; + +namespace Examine.Benchmarks +{ + public abstract class ExamineBaseTest + { + protected ILoggerFactory? LoggerFactory { get; private set; } + + public virtual void Setup() + { + LoggerFactory = CreateLoggerFactory(); + LoggerFactory.CreateLogger(typeof(ExamineBaseTest)).LogDebug("Initializing test"); + } + + public virtual void TearDown() => LoggerFactory!.Dispose(); + + public TestIndex GetTestIndex( + Directory luceneDir, + Directory luceneTaxonomyDir, + Analyzer analyzer, + FieldDefinitionCollection? fieldDefinitions = null, + IndexDeletionPolicy? indexDeletionPolicy = null, + IReadOnlyDictionary? indexValueTypesFactory = null, + double nrtTargetMaxStaleSec = 60, + double nrtTargetMinStaleSec = 1, + bool nrtEnabled = true) + => new TestIndex( + LoggerFactory!, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + FieldDefinitions = fieldDefinitions ?? new FieldDefinitionCollection(), + DirectoryFactory = GenericDirectoryFactory.FromExternallyManaged(_ => luceneDir, _ => luceneTaxonomyDir), + Analyzer = analyzer, + IndexDeletionPolicy = indexDeletionPolicy, + IndexValueTypesFactory = indexValueTypesFactory, +#if LocalBuild + NrtTargetMaxStaleSec = nrtTargetMaxStaleSec, + NrtTargetMinStaleSec = nrtTargetMinStaleSec, + NrtEnabled = nrtEnabled +#endif + })); + + //public TestIndex GetTestIndex( + // IndexWriter writer, + // double nrtTargetMaxStaleSec = 60, + // double nrtTargetMinStaleSec = 1) + // => new TestIndex( + // LoggerFactory, + // Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneIndexOptions + // { + // NrtTargetMaxStaleSec = nrtTargetMaxStaleSec, + // NrtTargetMinStaleSec = nrtTargetMinStaleSec + // }), + // writer); + + protected virtual ILoggerFactory CreateLoggerFactory() + => Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); + } +} diff --git a/src/Examine.Benchmarks/IndexVersionComparison.cs b/src/Examine.Benchmarks/IndexVersionComparison.cs new file mode 100644 index 000000000..ad11ab6ee --- /dev/null +++ b/src/Examine.Benchmarks/IndexVersionComparison.cs @@ -0,0 +1,48 @@ +using BenchmarkDotNet.Attributes; +using Examine.Lucene.Providers; +using Lucene.Net.Analysis.Standard; +using Microsoft.Extensions.Logging; + +namespace Examine.Benchmarks +{ + [Config(typeof(NugetConfig))] + [ThreadingDiagnoser] + [MemoryDiagnoser] + public class IndexVersionComparison : ExamineBaseTest + { + private readonly List _valueSets = InitTools.CreateValueSet(100); + private readonly StandardAnalyzer _analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + private ILogger? _logger; + private string? _tempBasePath; + private LuceneIndex? _indexer; + + [GlobalSetup] + public override void Setup() + { + base.Setup(); + + _logger = LoggerFactory!.CreateLogger(); + _tempBasePath = Path.Combine(Path.GetTempPath(), "ExamineTests"); + _indexer = InitTools.InitializeIndex(this, _tempBasePath, _analyzer, out _); + } + + [GlobalCleanup] + public override void TearDown() + { + _indexer!.Dispose(); + base.TearDown(); + System.IO.Directory.Delete(_tempBasePath!, true); + } + + [Benchmark] + public void IndexItemsNonAsync() => IndexItems(_indexer!, _valueSets); + +#if RELEASE + protected override ILoggerFactory CreateLoggerFactory() + => Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Information)); +#endif + + private static void IndexItems(LuceneIndex indexer, IEnumerable valueSets) + => indexer.IndexItems(valueSets); + } +} diff --git a/src/Examine.Benchmarks/InitTools.cs b/src/Examine.Benchmarks/InitTools.cs new file mode 100644 index 000000000..a40974c5a --- /dev/null +++ b/src/Examine.Benchmarks/InitTools.cs @@ -0,0 +1,43 @@ +using Lucene.Net.Analysis; +using Lucene.Net.Store; + +namespace Examine.Benchmarks +{ + internal class InitTools + { + public static TestIndex InitializeIndex( + ExamineBaseTest examineBaseTest, + string tempBasePath, + Analyzer analyzer, + out DirectoryInfo indexDir) + { + var tempPath = Path.Combine(tempBasePath, Guid.NewGuid().ToString()); + System.IO.Directory.CreateDirectory(tempPath); + indexDir = new DirectoryInfo(tempPath); + var luceneDirectory = FSDirectory.Open(indexDir); + var luceneTaxonomyDir = FSDirectory.Open(Path.Combine(tempPath, "Taxonomy")); + var indexer = examineBaseTest.GetTestIndex(luceneDirectory, luceneTaxonomyDir, analyzer); + return indexer; + } + + public static List CreateValueSet(int count) + { + var random = new Random(); + var valueSets = new List(); + + for (var i = 0; i < count; i++) + { + valueSets.Add(ValueSet.FromObject(Guid.NewGuid().ToString(), "content", + new + { + nodeName = "location" + (i % 2 == 0 ? "1" : "2"), + bodyText = Enumerable.Range(0, random.Next(10, 100)).Select(x => Guid.NewGuid().ToString()), + number = random.Next(0, 1000), + date = DateTime.Now.AddMinutes(random.Next(-1000, 1000)) + })); + } + + return valueSets; + } + } +} diff --git a/src/Examine.Benchmarks/NugetConfig.cs b/src/Examine.Benchmarks/NugetConfig.cs new file mode 100644 index 000000000..957aa8e33 --- /dev/null +++ b/src/Examine.Benchmarks/NugetConfig.cs @@ -0,0 +1,21 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; + +namespace Examine.Benchmarks +{ + public class NugetConfig : ManualConfig + { + public NugetConfig() + { + var baseJob = Job.ShortRun + .WithRuntime(CoreRuntime.Core80); + + AddJob(baseJob.WithId("Source")); + AddJob(baseJob.WithNuGet("Examine", "3.3.0").WithId("3.3.0").WithArguments([new MsBuildArgument("/p:LocalBuild=false")])); + AddJob(baseJob.WithNuGet("Examine", "3.2.1").WithId("3.2.1").WithArguments([new MsBuildArgument("/p:LocalBuild=false")])); + AddJob(baseJob.WithNuGet("Examine", "3.1.0").WithId("3.1.0").WithArguments([new MsBuildArgument("/p:LocalBuild=false")])); + AddJob(baseJob.WithNuGet("Examine", "3.0.1").WithId("3.0.1").WithArguments([new MsBuildArgument("/p:LocalBuild=false")])); + } + } +} diff --git a/src/Examine.Benchmarks/Program.cs b/src/Examine.Benchmarks/Program.cs new file mode 100644 index 000000000..51efe3c44 --- /dev/null +++ b/src/Examine.Benchmarks/Program.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BenchmarkDotNet.Running; +using Microsoft.Diagnostics.Tracing.Parsers.Kernel; + +namespace Examine.Benchmarks +{ + public class Program + { +#if RELEASE + public static void Main(string[] args) => + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); +#else + public static async Task Main(string[] args) + { + var bench = new SearchVersionComparison(); + try + { + bench.Setup(); + //await Threads100(bench); + await Threads1(bench); + } + finally + { + bench.TearDown(); + } + } +#endif + // Call your function here. + +#if LocalBuild + private static async Task Threads100(ConcurrentSearchBenchmarks bench) + { + bench.ThreadCount = 50; + //bench.MaxResults = 10; + + for (var i = 0; i < 100; i++) + { + await bench.ExamineStandard(); + } + } + + private static async Task Threads1(SearchVersionComparison bench) + { + bench.ThreadCount = 1; + //bench.MaxResults = 10; + + for (var i = 0; i < 100; i++) + { + await bench.ConcurrentSearch(); + } + } +#endif + } +} diff --git a/src/Examine.Benchmarks/SearchVersionComparison.cs b/src/Examine.Benchmarks/SearchVersionComparison.cs new file mode 100644 index 000000000..8499c54bb --- /dev/null +++ b/src/Examine.Benchmarks/SearchVersionComparison.cs @@ -0,0 +1,79 @@ +using BenchmarkDotNet.Attributes; +using Examine.Lucene.Providers; +using Lucene.Net.Analysis.Standard; +using Microsoft.Extensions.Logging; + +namespace Examine.Benchmarks +{ + [Config(typeof(NugetConfig))] + [HideColumns("Arguments", "StdDev", "Error", "NuGetReferences")] + [MemoryDiagnoser] + public class SearchVersionComparison : ExamineBaseTest + { + private readonly List _valueSets = InitTools.CreateValueSet(10000); + private readonly StandardAnalyzer _analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + private ILogger? _logger; + private string? _tempBasePath; + private LuceneIndex? _indexer; + + [GlobalSetup] + public override void Setup() + { + base.Setup(); + + _logger = LoggerFactory!.CreateLogger(); + _tempBasePath = Path.Combine(Path.GetTempPath(), "ExamineTests"); + _indexer = InitTools.InitializeIndex(this, _tempBasePath, _analyzer, out _); + _indexer!.IndexItems(_valueSets); + + _logger.LogInformation("Indexed {DocumentCount} documents", _valueSets.Count); + } + + [GlobalCleanup] + public override void TearDown() + { + _indexer!.Dispose(); + base.TearDown(); + Directory.Delete(_tempBasePath!, true); + } + + [Params(1, 25, 100)] + public int ThreadCount { get; set; } + + [Benchmark] + public async Task ConcurrentSearch() + { + var tasks = new List(); + + for (var i = 0; i < ThreadCount; i++) + { + tasks.Add(new Task(() => + { + // always resolve the searcher from the indexer + var searcher = _indexer!.Searcher; + + var query = searcher.CreateQuery().Field("nodeName", "location1"); + var results = query.Execute(); + + // enumerate (forces the result to execute) + var logOutput = "ThreadID: " + Thread.CurrentThread.ManagedThreadId + ", Results: " + string.Join(',', results.Select(x => $"{x.Id}-{x.Values.Count}-{x.Score}").ToArray()); + _logger!.LogDebug(logOutput); + + //_logger!.LogInformation("Results: {Results}", results.TotalItemCount); + })); + } + + foreach (var task in tasks) + { + task.Start(); + } + + await Task.WhenAll(tasks); + } + +#if RELEASE + protected override ILoggerFactory CreateLoggerFactory() + => Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Information)); +#endif + } +} diff --git a/src/Examine.Benchmarks/TestIndex.cs b/src/Examine.Benchmarks/TestIndex.cs new file mode 100644 index 000000000..7eaebf57e --- /dev/null +++ b/src/Examine.Benchmarks/TestIndex.cs @@ -0,0 +1,35 @@ +using Examine.Lucene; +using Examine.Lucene.Providers; +using Lucene.Net.Index; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Examine.Benchmarks +{ + public class TestIndex : LuceneIndex + { + public const string TestIndexName = "testIndexer"; + + public TestIndex(ILoggerFactory loggerFactory, IOptionsMonitor options) + : base(loggerFactory, TestIndexName, options) + { + RunAsync = false; + } + + //public TestIndex(ILoggerFactory loggerFactory, IOptionsMonitor options, IndexWriter writer) + // : base(loggerFactory, TestIndexName, options, writer) + //{ + // RunAsync = false; + //} + + public static IEnumerable AllData() + { + var data = new List(); + for (var i = 0; i < 100; i++) + { + data.Add(ValueSet.FromObject(i.ToString(), "category" + (i % 2), new { item1 = "value" + i, item2 = "value" + i })); + } + return data; + } + } +} diff --git a/src/Examine.Core/Examine.Core.csproj b/src/Examine.Core/Examine.Core.csproj index 9d22f986a..2158feb43 100644 --- a/src/Examine.Core/Examine.Core.csproj +++ b/src/Examine.Core/Examine.Core.csproj @@ -6,8 +6,6 @@ Examine is an abstraction for indexing and search operations with implementations such as Lucene.Net examine search index True - enable - 9 @@ -29,8 +27,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/src/Examine.Core/ExamineManager.cs b/src/Examine.Core/ExamineManager.cs index a057ea101..1eddd5777 100644 --- a/src/Examine.Core/ExamineManager.cs +++ b/src/Examine.Core/ExamineManager.cs @@ -14,12 +14,12 @@ public class ExamineManager : IDisposable, IExamineManager /// public ExamineManager(IEnumerable indexes, IEnumerable searchers) { - foreach(var i in indexes) + foreach (var i in indexes) { AddIndex(i); } - foreach(var s in searchers) + foreach (var s in searchers) { AddSearcher(s); } @@ -30,17 +30,13 @@ public ExamineManager(IEnumerable indexes, IEnumerable search /// public bool TryGetSearcher(string searcherName, -#if !NETSTANDARD2_0 [MaybeNullWhen(false)] -#endif out ISearcher searcher) => (searcher = _searchers.TryGetValue(searcherName, out var s) ? s : null) != null; /// public bool TryGetIndex(string indexName, -#if !NETSTANDARD2_0 [MaybeNullWhen(false)] -#endif out IIndex index) => (index = _indexers.TryGetValue(indexName, out var i) ? i : null) != null; @@ -86,7 +82,7 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - Stop(false); + // NOTE: Legacy - this used to dispose indexes, but that is managed by DI now Stop(true); } @@ -124,21 +120,6 @@ public virtual void Stop(bool immediate) // no strange lucene background thread stuff causes issues here. } } - else - { - try - { - foreach (var indexer in Indexes.OfType()) - { - indexer.Dispose(); - } - } - catch (Exception) - { - //an exception shouldn't occur but if so we need to terminate - Stop(true); - } - } } } } diff --git a/src/Examine.Core/FieldDefinition.cs b/src/Examine.Core/FieldDefinition.cs index a8804a481..8ed24c7ac 100644 --- a/src/Examine.Core/FieldDefinition.cs +++ b/src/Examine.Core/FieldDefinition.cs @@ -5,7 +5,7 @@ namespace Examine /// /// Defines a field to be indexed /// - public struct FieldDefinition : IEquatable + public readonly struct FieldDefinition : IEquatable { /// /// Constructor @@ -18,12 +18,10 @@ public FieldDefinition(string name, string type) { throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); } - if (string.IsNullOrWhiteSpace(type)) { throw new ArgumentException("Value cannot be null or whitespace.", nameof(type)); } - Name = name; Type = type; } diff --git a/src/Examine.Core/IExamineManager.cs b/src/Examine.Core/IExamineManager.cs index c2583f246..0c6806a69 100644 --- a/src/Examine.Core/IExamineManager.cs +++ b/src/Examine.Core/IExamineManager.cs @@ -15,7 +15,7 @@ public interface IExamineManager /// /// This returns all config based indexes and indexers registered in code /// - IEnumerable Indexes { get; } + public IEnumerable Indexes { get; } /// /// Gets a list of all manually configured search providers @@ -23,12 +23,12 @@ public interface IExamineManager /// /// This returns only those searchers explicitly registered with AddExamineSearcher or config based searchers /// - IEnumerable RegisteredSearchers { get; } + public IEnumerable RegisteredSearchers { get; } /// /// Disposes the /// - void Dispose(); + public void Dispose(); /// /// Returns an indexer by name @@ -36,10 +36,8 @@ public interface IExamineManager /// /// /// true if the index was found by name - bool TryGetIndex(string indexName, -#if !NETSTANDARD2_0 + public bool TryGetIndex(string indexName, [MaybeNullWhen(false)] -#endif out IIndex index); /// @@ -50,11 +48,9 @@ bool TryGetIndex(string indexName, /// /// true if the searcher was found by name /// - bool TryGetSearcher(string searcherName, -#if !NETSTANDARD2_0 + public bool TryGetSearcher(string searcherName, [MaybeNullWhen(false)] -#endif - out ISearcher searcher); + out ISearcher searcher); } } diff --git a/src/Examine.Core/ISearcher.cs b/src/Examine.Core/ISearcher.cs index 3ad4e9982..bae5fe0fd 100644 --- a/src/Examine.Core/ISearcher.cs +++ b/src/Examine.Core/ISearcher.cs @@ -10,7 +10,7 @@ public interface ISearcher /// /// The searchers name /// - string Name { get; } + public string Name { get; } /// /// Searches the index @@ -18,7 +18,7 @@ public interface ISearcher /// The search text or a native query /// /// Search Results - ISearchResults Search(string searchText, QueryOptions? options = null); + public ISearchResults Search(string searchText, QueryOptions? options = null); /// /// Creates a search criteria instance as required by the implementation @@ -28,6 +28,6 @@ public interface ISearcher /// /// An instance of /// - IQuery CreateQuery(string? category = null, BooleanOperation defaultOperation = BooleanOperation.And); + public IQuery CreateQuery(string? category = null, BooleanOperation defaultOperation = BooleanOperation.And); } } diff --git a/src/Examine.Core/IndexOperation.cs b/src/Examine.Core/IndexOperation.cs index 7ddbd4ea7..5d7122c83 100644 --- a/src/Examine.Core/IndexOperation.cs +++ b/src/Examine.Core/IndexOperation.cs @@ -3,7 +3,7 @@ namespace Examine /// /// Represents an indexing operation (either add/remove) /// - public struct IndexOperation + public readonly struct IndexOperation { /// /// Initializes a new instance of the class. @@ -27,4 +27,4 @@ public IndexOperation(ValueSet valueSet, IndexOperationType operation) /// public IndexOperationType Operation { get; } } -} \ No newline at end of file +} diff --git a/src/Examine.Core/OrderedDictionary.cs b/src/Examine.Core/OrderedDictionary.cs index bbca697fb..f4e72c026 100644 --- a/src/Examine.Core/OrderedDictionary.cs +++ b/src/Examine.Core/OrderedDictionary.cs @@ -69,13 +69,8 @@ public void Add(TKey key, TVal value) } /// -#pragma warning disable CS8767 // Nullability of reference types in type of parameter doesn't match implicitly implemented member (possibly because of nullability attributes). - // Justification for warning disabled: IDictionary is missing [MaybeNullWhen(false)] in Netstandard 2.1 public bool TryGetValue(TKey key, -#pragma warning restore CS8767 // Nullability of reference types in type of parameter doesn't match implicitly implemented member (possibly because of nullability attributes). -#if !NETSTANDARD2_0 [MaybeNullWhen(false)] -#endif out TVal value) { if (Dictionary == null) diff --git a/src/Examine.Core/PublicAPI.Shipped.txt b/src/Examine.Core/PublicAPI.Shipped.txt index 0c6962085..c9886c7fd 100644 --- a/src/Examine.Core/PublicAPI.Shipped.txt +++ b/src/Examine.Core/PublicAPI.Shipped.txt @@ -27,7 +27,6 @@ const Examine.FieldDefinitionTypes.Integer = "int" -> string! const Examine.FieldDefinitionTypes.InvariantCultureIgnoreCase = "invariantcultureignorecase" -> string! const Examine.FieldDefinitionTypes.Long = "long" -> string! const Examine.FieldDefinitionTypes.Raw = "raw" -> string! -const Examine.Search.QueryOptions.DefaultMaxResults = 500 -> int Examine.BaseIndexProvider Examine.BaseIndexProvider.BaseIndexProvider(Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, string! name, Microsoft.Extensions.Options.IOptionsMonitor! indexOptions) -> void Examine.BaseIndexProvider.DeleteFromIndex(System.Collections.Generic.IEnumerable! itemIds) -> void @@ -259,18 +258,13 @@ Examine.SearchResult.this[int resultIndex].get -> System.Collections.Generic.Key Examine.SearchResult.this[string! key].get -> string? Examine.SearchResult.Values.get -> System.Collections.Generic.IReadOnlyDictionary! Examine.ValueSet -Examine.ValueSet.Category.get -> string? Examine.ValueSet.Clone() -> Examine.ValueSet! Examine.ValueSet.GetValue(string! key) -> object? Examine.ValueSet.GetValues(string! key) -> System.Collections.Generic.IEnumerable! Examine.ValueSet.Id.get -> string! -Examine.ValueSet.ItemType.get -> string? -Examine.ValueSet.Values.get -> System.Collections.Generic.IReadOnlyDictionary!>? -Examine.ValueSet.ValueSet(string! id) -> void Examine.ValueSet.ValueSet(string! id, string! category, string! itemType, System.Collections.Generic.IDictionary! values) -> void Examine.ValueSet.ValueSet(string! id, string! category, System.Collections.Generic.IDictionary! values) -> void Examine.ValueSet.ValueSet(string! id, string! category, System.Collections.Generic.IDictionary!>! values) -> void -Examine.ValueSet.ValueSet(string! id, string? category, string? itemType, System.Collections.Generic.IDictionary!>! values) -> void Examine.ValueSetValidationResult Examine.ValueSetValidationResult.Status.get -> Examine.ValueSetValidationStatus Examine.ValueSetValidationResult.ValueSet.get -> Examine.ValueSet! diff --git a/src/Examine.Core/PublicAPI.Unshipped.txt b/src/Examine.Core/PublicAPI.Unshipped.txt index b43c11e77..e218bacf3 100644 --- a/src/Examine.Core/PublicAPI.Unshipped.txt +++ b/src/Examine.Core/PublicAPI.Unshipped.txt @@ -89,4 +89,11 @@ Examine.Search.Int64Range.MaxInclusive.get -> bool Examine.Search.Int64Range.Min.get -> long Examine.Search.Int64Range.MinInclusive.get -> bool Examine.Search.OrderingExtensions +Examine.ValueSet.Category.get -> string! +Examine.ValueSet.ItemType.get -> string! +Examine.ValueSet.Values.get -> System.Collections.Generic.IReadOnlyDictionary!>! +Examine.ValueSet.ValueSet(string! id) -> void +Examine.ValueSet.ValueSet(string! id, string! category, string! itemType, System.Collections.Generic.IDictionary!>! values) -> void static Examine.Search.OrderingExtensions.WithFacets(this Examine.Search.IOrdering! ordering, System.Action! facets) -> Examine.Search.IQueryExecutor! +const Examine.Search.QueryOptions.AbsoluteMaxResults = 10000 -> int +const Examine.Search.QueryOptions.DefaultMaxResults = 100 -> int diff --git a/src/Examine.Core/Search/ExamineValue.cs b/src/Examine.Core/Search/ExamineValue.cs index 06cb5668e..7c99ce2ac 100644 --- a/src/Examine.Core/Search/ExamineValue.cs +++ b/src/Examine.Core/Search/ExamineValue.cs @@ -3,7 +3,7 @@ namespace Examine.Search { /// - public struct ExamineValue : IExamineValue + public readonly struct ExamineValue : IExamineValue { /// public ExamineValue(Examineness vagueness, string value) @@ -27,6 +27,5 @@ public ExamineValue(Examineness vagueness, string value, float level) /// public float Level { get; } - } } diff --git a/src/Examine.Core/Search/FacetResult.cs b/src/Examine.Core/Search/FacetResult.cs index 4df8e4956..834deb3b7 100644 --- a/src/Examine.Core/Search/FacetResult.cs +++ b/src/Examine.Core/Search/FacetResult.cs @@ -9,12 +9,9 @@ namespace Examine.Search public class FacetResult : IFacetResult { private readonly IEnumerable _values; -#if NETSTANDARD2_1 + [AllowNull] private IDictionary _dictValues; -#else - private IDictionary? _dictValues; -#endif /// public FacetResult(IEnumerable values) @@ -25,9 +22,7 @@ public FacetResult(IEnumerable values) /// public IEnumerator GetEnumerator() => _values.GetEnumerator(); -#if !NETSTANDARD2_0 && !NETSTANDARD2_1 [MemberNotNull(nameof(_dictValues))] -#endif private void SetValuesDictionary() => _dictValues ??= _values.ToDictionary(src => src.Label, src => src); /// diff --git a/src/Examine.Core/Search/IFacetResult.cs b/src/Examine.Core/Search/IFacetResult.cs index 9ccd174a3..59b7ce15b 100644 --- a/src/Examine.Core/Search/IFacetResult.cs +++ b/src/Examine.Core/Search/IFacetResult.cs @@ -12,14 +12,14 @@ public interface IFacetResult : IEnumerable /// /// /// - IFacetValue? Facet(string label); + public IFacetValue? Facet(string label); /// - /// Trys to get a facet value for a label + /// Try to get a facet value for a label /// /// /// /// - bool TryGetFacet(string label, out IFacetValue? facetValue); + public bool TryGetFacet(string label, out IFacetValue? facetValue); } } diff --git a/src/Examine.Core/Search/QueryOptions.cs b/src/Examine.Core/Search/QueryOptions.cs index 587468a0b..27ac60e4c 100644 --- a/src/Examine.Core/Search/QueryOptions.cs +++ b/src/Examine.Core/Search/QueryOptions.cs @@ -8,9 +8,17 @@ namespace Examine.Search public class QueryOptions { /// - /// The default maximum ammount of results + /// The absolute maximum results returned from a query /// - public const int DefaultMaxResults = 500; + /// + /// This limit is applied to prevent excessive resource usage and potential performance issues. + /// + public const int AbsoluteMaxResults = 10000; + + /// + /// The default maximum amount of results + /// + public const int DefaultMaxResults = 100; /// /// Creates a with the specified parameters diff --git a/src/Examine.Core/Search/SortableField.cs b/src/Examine.Core/Search/SortableField.cs index 2ef14efa4..14665a03b 100644 --- a/src/Examine.Core/Search/SortableField.cs +++ b/src/Examine.Core/Search/SortableField.cs @@ -1,9 +1,9 @@ -namespace Examine.Search +namespace Examine.Search { /// /// Represents a field used to sort results /// - public struct SortableField + public readonly struct SortableField { /// /// The field name to sort by @@ -36,4 +36,4 @@ public SortableField(string fieldName, SortType sortType) SortType = sortType; } } -} \ No newline at end of file +} diff --git a/src/Examine.Core/ValueSet.cs b/src/Examine.Core/ValueSet.cs index 06c876354..db0088e36 100644 --- a/src/Examine.Core/ValueSet.cs +++ b/src/Examine.Core/ValueSet.cs @@ -10,6 +10,8 @@ namespace Examine /// public class ValueSet { + private static readonly IReadOnlyDictionary> Empty = new Dictionary>(); + /// /// The id of the object to be indexed /// @@ -21,17 +23,17 @@ public class ValueSet /// /// Used to categorize the item in the index (in umbraco terms this would be content vs media) /// - public string? Category { get; } + public string Category { get; } = string.Empty; /// /// The item's node type (in umbraco terms this would be the doc type alias) /// - public string? ItemType { get; } + public string ItemType { get; } = string.Empty; /// /// The values to be indexed /// - public IReadOnlyDictionary>? Values { get; } + public IReadOnlyDictionary> Values { get; } = Empty; /// /// Constructor that only specifies an ID @@ -114,17 +116,17 @@ public ValueSet(string id, string category, IDictionary /// - public ValueSet(string id, string? category, string? itemType, IDictionary> values) + public ValueSet(string id, string category, string itemType, IDictionary> values) : this(id, category, itemType, values.ToDictionary(x => x.Key, x => (IReadOnlyList)x.Value.ToList())) { } - private ValueSet(string id, string? category, string? itemType, IReadOnlyDictionary>? values) + private ValueSet(string id, string category, string itemType, IReadOnlyDictionary> values) { Id = id; Category = category; ItemType = itemType; - Values = values?.ToDictionary(x => x.Key, x => (IReadOnlyList)x.Value.ToList()) ?? default; + Values = values.ToDictionary(x => x.Key, x => (IReadOnlyList)x.Value.ToList()); } /// diff --git a/src/Examine.Core/ValueSetValidationResult.cs b/src/Examine.Core/ValueSetValidationResult.cs index dbdb2ada6..015fbed1d 100644 --- a/src/Examine.Core/ValueSetValidationResult.cs +++ b/src/Examine.Core/ValueSetValidationResult.cs @@ -3,7 +3,7 @@ namespace Examine /// /// Represents a value set validation result /// - public struct ValueSetValidationResult + public readonly struct ValueSetValidationResult { /// public ValueSetValidationResult(ValueSetValidationStatus status, ValueSet valueSet) diff --git a/src/Examine.Host/Examine.csproj b/src/Examine.Host/Examine.csproj index ae0f8a7d4..c28cfaee2 100644 --- a/src/Examine.Host/Examine.csproj +++ b/src/Examine.Host/Examine.csproj @@ -23,12 +23,13 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/src/Examine.Host/ExamineLuceneIndexConfiguration.cs b/src/Examine.Host/ExamineLuceneIndexConfiguration.cs deleted file mode 100644 index cdab44c74..000000000 --- a/src/Examine.Host/ExamineLuceneIndexConfiguration.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System.Collections.Generic; -using Examine.Lucene; -using Examine.Lucene.Directories; -using Examine.Lucene.Providers; -using Lucene.Net.Analysis; -using Lucene.Net.Facet; - -namespace Examine -{ - /// - /// Examine Lucene Index Configuration - /// - /// - /// - public class ExamineLuceneIndexConfiguration - where TIndex : LuceneIndex - where TDirectoryFactory : class, IDirectoryFactory - { - /// - /// Constructor - /// - /// Index Name - public ExamineLuceneIndexConfiguration(string name) - { - Name = name; - } - - /// - /// Index Name - /// - public string Name { get; } - - /// - /// Index Field Definitions - /// - public FieldDefinitionCollection? FieldDefinitions { get; set; } - - /// - /// Index Default Analyzer - /// - public Analyzer? Analyzer { get; set; } - - /// - /// Index Value Set Validator - /// - public IValueSetValidator? Validator { get; set; } - - /// - /// Index Value Type Factory - /// - public IReadOnlyDictionary? IndexValueTypesFactory { get; set; } - - /// - /// Index Facet Config - /// - public FacetsConfig? FacetsConfig { get; set; } - - /// - /// Whether to use Taxonomy Index - /// - public bool UseTaxonomyIndex { get; set; } - } -} diff --git a/src/Examine.Host/ExamineLuceneMultiSearcherConfiguration.cs b/src/Examine.Host/ExamineLuceneMultiSearcherConfiguration.cs deleted file mode 100644 index 13e759106..000000000 --- a/src/Examine.Host/ExamineLuceneMultiSearcherConfiguration.cs +++ /dev/null @@ -1,43 +0,0 @@ - -using Lucene.Net.Analysis; -using Lucene.Net.Facet; - -namespace Examine -{ - /// - /// Examine Lucene MultiSearcher Configuration - /// - public class ExamineLuceneMultiSearcherConfiguration - { - /// - /// Constructor - /// - /// Searcher Name - /// Index Names to search - public ExamineLuceneMultiSearcherConfiguration(string name, string[] indexNames) - { - Name = name; - IndexNames = indexNames; - } - - /// - /// Searcher Name - /// - public string Name { get; } - - /// - /// Index Names to search - /// - public string[] IndexNames { get; } - - /// - /// Facet Configuration - /// - public FacetsConfig? FacetConfiguration { get; set; } - - /// - /// Search Analyzer - /// - public Analyzer? Analyzer { get; set; } - } -} diff --git a/src/Examine.Host/PublicAPI.Shipped.txt b/src/Examine.Host/PublicAPI.Shipped.txt index 525420199..3c1185299 100644 --- a/src/Examine.Host/PublicAPI.Shipped.txt +++ b/src/Examine.Host/PublicAPI.Shipped.txt @@ -9,8 +9,3 @@ Examine.IApplicationRoot Examine.IApplicationRoot.ApplicationRoot.get -> System.IO.DirectoryInfo! Examine.ServicesCollectionExtensions static Examine.ServicesCollectionExtensions.AddExamine(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.IO.DirectoryInfo? appRootDirectory = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, Examine.FieldDefinitionCollection? fieldDefinitions = null, Lucene.Net.Analysis.Analyzer? analyzer = null, Examine.IValueSetValidator? validator = null, System.Collections.Generic.IReadOnlyDictionary? indexValueTypesFactory = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, Examine.FieldDefinitionCollection? fieldDefinitions = null, Lucene.Net.Analysis.Analyzer? analyzer = null, Examine.IValueSetValidator? validator = null, System.Collections.Generic.IReadOnlyDictionary? indexValueTypesFactory = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, Examine.FieldDefinitionCollection? fieldDefinitions = null, Lucene.Net.Analysis.Analyzer? analyzer = null, Examine.IValueSetValidator? validator = null, System.Collections.Generic.IReadOnlyDictionary? indexValueTypesFactory = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineLuceneMultiSearcher(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, string![]! indexNames, Lucene.Net.Analysis.Analyzer? analyzer = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineSearcher(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, System.Func!>! parameterFactory) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! diff --git a/src/Examine.Host/PublicAPI.Unshipped.txt b/src/Examine.Host/PublicAPI.Unshipped.txt index ed0687448..5616cbe8b 100644 --- a/src/Examine.Host/PublicAPI.Unshipped.txt +++ b/src/Examine.Host/PublicAPI.Unshipped.txt @@ -1,28 +1,5 @@ #nullable enable -Examine.ExamineLuceneIndexConfiguration -Examine.ExamineLuceneIndexConfiguration.Analyzer.get -> Lucene.Net.Analysis.Analyzer? -Examine.ExamineLuceneIndexConfiguration.Analyzer.set -> void -Examine.ExamineLuceneIndexConfiguration.ExamineLuceneIndexConfiguration(string! name) -> void -Examine.ExamineLuceneIndexConfiguration.FacetsConfig.get -> Lucene.Net.Facet.FacetsConfig? -Examine.ExamineLuceneIndexConfiguration.FacetsConfig.set -> void -Examine.ExamineLuceneIndexConfiguration.FieldDefinitions.get -> Examine.FieldDefinitionCollection? -Examine.ExamineLuceneIndexConfiguration.FieldDefinitions.set -> void -Examine.ExamineLuceneIndexConfiguration.IndexValueTypesFactory.get -> System.Collections.Generic.IReadOnlyDictionary? -Examine.ExamineLuceneIndexConfiguration.IndexValueTypesFactory.set -> void -Examine.ExamineLuceneIndexConfiguration.Name.get -> string! -Examine.ExamineLuceneIndexConfiguration.UseTaxonomyIndex.get -> bool -Examine.ExamineLuceneIndexConfiguration.UseTaxonomyIndex.set -> void -Examine.ExamineLuceneIndexConfiguration.Validator.get -> Examine.IValueSetValidator? -Examine.ExamineLuceneIndexConfiguration.Validator.set -> void -Examine.ExamineLuceneMultiSearcherConfiguration -Examine.ExamineLuceneMultiSearcherConfiguration.Analyzer.get -> Lucene.Net.Analysis.Analyzer? -Examine.ExamineLuceneMultiSearcherConfiguration.Analyzer.set -> void -Examine.ExamineLuceneMultiSearcherConfiguration.ExamineLuceneMultiSearcherConfiguration(string! name, string![]! indexNames) -> void -Examine.ExamineLuceneMultiSearcherConfiguration.FacetConfiguration.get -> Lucene.Net.Facet.FacetsConfig? -Examine.ExamineLuceneMultiSearcherConfiguration.FacetConfiguration.set -> void -Examine.ExamineLuceneMultiSearcherConfiguration.IndexNames.get -> string![]! -Examine.ExamineLuceneMultiSearcherConfiguration.Name.get -> string! -static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, System.Action!>! configuration) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, System.Action!>! configuration) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, System.Action!>! configuration) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! -static Examine.ServicesCollectionExtensions.AddExamineLuceneMultiSearcher(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, string![]! indexNames, System.Action? configuration = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, System.Action? configuration = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, System.Action? configuration = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Examine.ServicesCollectionExtensions.AddExamineLuceneIndex(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, System.Action? configuration = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! +static Examine.ServicesCollectionExtensions.AddExamineLuceneMultiSearcher(this Microsoft.Extensions.DependencyInjection.IServiceCollection! serviceCollection, string! name, string![]! indexNames, System.Action? configuration = null) -> Microsoft.Extensions.DependencyInjection.IServiceCollection! diff --git a/src/Examine.Host/ServicesCollectionExtensions.cs b/src/Examine.Host/ServicesCollectionExtensions.cs index 4d7c72614..2ca9ee106 100644 --- a/src/Examine.Host/ServicesCollectionExtensions.cs +++ b/src/Examine.Host/ServicesCollectionExtensions.cs @@ -18,82 +18,18 @@ namespace Examine /// public static class ServicesCollectionExtensions { - /// - /// Registers a file system based Lucene Examine index - /// - [Obsolete("To remove in Examine V5")] - public static IServiceCollection AddExamineLuceneIndex( - this IServiceCollection serviceCollection, - string name, - FieldDefinitionCollection? fieldDefinitions = null, - Analyzer? analyzer = null, - IValueSetValidator? validator = null, - IReadOnlyDictionary? indexValueTypesFactory = null) - => serviceCollection.AddExamineLuceneIndex(name, fieldDefinitions, analyzer, validator, indexValueTypesFactory); - - /// - /// Registers a file system based Lucene Examine index - /// - [Obsolete("To remove in Examine V5")] - public static IServiceCollection AddExamineLuceneIndex( - this IServiceCollection serviceCollection, - string name, - FieldDefinitionCollection? fieldDefinitions = null, - Analyzer? analyzer = null, - IValueSetValidator? validator = null, - IReadOnlyDictionary? indexValueTypesFactory = null) - where TIndex : LuceneIndex - { - Action> config = opt => - { - opt.FieldDefinitions = fieldDefinitions; - opt.Analyzer = analyzer; - opt.UseTaxonomyIndex = false; - opt.FacetsConfig = null; - opt.IndexValueTypesFactory = indexValueTypesFactory; - opt.Validator = validator; - }; - return serviceCollection.AddExamineLuceneIndex(name, config); - } - - /// - /// Registers an Examine index - /// - [Obsolete("To remove in Examine V5")] - public static IServiceCollection AddExamineLuceneIndex( - this IServiceCollection serviceCollection, - string name, - FieldDefinitionCollection? fieldDefinitions = null, - Analyzer? analyzer = null, - IValueSetValidator? validator = null, - IReadOnlyDictionary? indexValueTypesFactory = null) - where TIndex : LuceneIndex - where TDirectoryFactory : class, IDirectoryFactory - { - Action> config = opt => - { - opt.FieldDefinitions = fieldDefinitions; - opt.Analyzer = analyzer; - opt.UseTaxonomyIndex = false; - opt.FacetsConfig = null; - opt.IndexValueTypesFactory = indexValueTypesFactory; - opt.Validator = validator; - }; - return serviceCollection.AddExamineLuceneIndex(name, config); - } - /// /// Registers an Examine index /// public static IServiceCollection AddExamineLuceneIndex( this IServiceCollection serviceCollection, string name, - Action> configuration) + Action? configuration = null) where TIndex : LuceneIndex where TDirectoryFactory : class, IDirectoryFactory { - var config = new ExamineLuceneIndexConfiguration(name); - configuration.Invoke(config); + var config = new LuceneDirectoryIndexOptions(); + configuration?.Invoke(config); // This is the long way to add IOptions but gives us access to the // services collection which we need to get the dir factory @@ -108,13 +44,11 @@ public static IServiceCollection AddExamineLuceneIndex(); options.FacetsConfig = config.FacetsConfig ?? new FacetsConfig(); - options.UseTaxonomyIndex = config.UseTaxonomyIndex; })); return serviceCollection.AddSingleton(services => { - var options - = services.GetRequiredService>(); + var options = services.GetRequiredService>(); var index = ActivatorUtilities.CreateInstance( services, @@ -130,7 +64,7 @@ var options public static IServiceCollection AddExamineLuceneIndex( this IServiceCollection serviceCollection, string name, - Action> configuration) => serviceCollection.AddExamineLuceneIndex(name, configuration); + Action? configuration = null) => serviceCollection.AddExamineLuceneIndex(name, configuration); /// /// Registers a file system based Lucene Examine index @@ -138,88 +72,47 @@ public static IServiceCollection AddExamineLuceneIndex( public static IServiceCollection AddExamineLuceneIndex( this IServiceCollection serviceCollection, string name, - Action> configuration) + Action? configuration = null) where TIndex : LuceneIndex => serviceCollection.AddExamineLuceneIndex(name, configuration); /// - /// Registers a standalone Examine searcher + /// Registers a Lucene multi index searcher /// - /// - /// - /// - /// - /// A factory to fullfill the custom searcher construction parameters excluding the name that are not already registerd in DI. - /// - /// - public static IServiceCollection AddExamineSearcher( - this IServiceCollection serviceCollection, - string name, - Func> parameterFactory) - where TSearcher : ISearcher - => serviceCollection.AddTransient(services => - { - var parameters = parameterFactory(services); - parameters.Insert(0, name); - - var searcher = ActivatorUtilities.CreateInstance( - services, - parameters.ToArray()); - - return searcher; - }); - - /// - /// Registers a lucene multi index searcher - /// - [Obsolete("Will be removed in Examine V5")] public static IServiceCollection AddExamineLuceneMultiSearcher( this IServiceCollection serviceCollection, string name, string[] indexNames, - Analyzer? analyzer = null) + Action? configuration = null) { - var cfg = new Action(opt => + var config = new LuceneMultiSearcherOptions { - opt.Analyzer = analyzer; - opt.FacetConfiguration = default; - }); - return AddExamineLuceneMultiSearcher(serviceCollection, name, indexNames, cfg); - } + IndexNames = indexNames + }; + configuration?.Invoke(config); - /// - /// Registers a lucene multi index searcher - /// - public static IServiceCollection AddExamineLuceneMultiSearcher( - this IServiceCollection serviceCollection, - string name, - string[] indexNames, - Action? configuration = null) - { - var cfg = new ExamineLuceneMultiSearcherConfiguration(name, indexNames); - configuration?.Invoke(cfg); - return serviceCollection.AddExamineSearcher(name, s => + // This is the long way to add IOptions but gives us access to the + // services collection which we need to get the dir factory + serviceCollection.AddSingleton>( + services => new ConfigureNamedOptions( + name, + (options) => { - var matchedIndexes = s.GetServices() - .Where(x => indexNames.Contains(x.Name)); - - var parameters = new List - { - matchedIndexes, - }; - - if (cfg.FacetConfiguration != null) - { - parameters.Add(cfg.FacetConfiguration); - } + options.Analyzer = config.Analyzer; + options.FacetConfiguration = config.FacetConfiguration; + })); - if (cfg.Analyzer != null) - { - parameters.Add(cfg.Analyzer); - } + // Transient I think because of how the search context is created, it can't hang on to it. + return serviceCollection.AddTransient(s => + { + var namedOptions = s.GetRequiredService>().Get(name); + var matchedIndexes = s.GetServices().Where(x => namedOptions.IndexNames.Contains(x.Name)); + var searcher = ActivatorUtilities.CreateInstance( + s, + matchedIndexes); - return parameters; - }); + return searcher; + }); } /// diff --git a/src/Examine.Lucene/Directories/DirectoryFactory.cs b/src/Examine.Lucene/Directories/DirectoryFactory.cs index b6c5e4c14..a136487a1 100644 --- a/src/Examine.Lucene/Directories/DirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/DirectoryFactory.cs @@ -8,34 +8,46 @@ namespace Examine.Lucene.Directories /// /// Represents a generic directory factory /// - public class GenericDirectoryFactory : DirectoryFactoryBase + public class GenericDirectoryFactory : IDirectoryFactory { private readonly Func _factory; - private readonly Func? _taxonomyDirectoryFactory; + private readonly Func _taxonomyDirectoryFactory; /// - /// Creates a an instance of + /// Creates an instance of /// - /// The factory - [Obsolete("To remove in Examine V5")] - public GenericDirectoryFactory(Func factory) + public GenericDirectoryFactory( + Func factory, + Func taxonomyDirectoryFactory) + : this(false, factory, taxonomyDirectoryFactory) { - _factory = factory; } - /// - /// Creates a an instance of - /// - /// The factory - /// The taxonomy directory factory - public GenericDirectoryFactory(Func factory, Func taxonomyDirectoryFactory) + private GenericDirectoryFactory( + bool externallyManaged, + Func factory, + Func taxonomyDirectoryFactory) { + ExternallyManaged = externallyManaged; _factory = factory; _taxonomyDirectoryFactory = taxonomyDirectoryFactory; } + /// + /// Creates a instance with externally managed directories. + /// + internal static GenericDirectoryFactory FromExternallyManaged( + Func factory, + Func taxonomyDirectoryFactory) => + new(true, factory, taxonomyDirectoryFactory); + + /// + /// When set to true, indicates that the directory is managed externally and will be disposed of by the caller, not the index. + /// + internal bool ExternallyManaged { get; } + /// - protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) + public Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) { var dir = _factory(luceneIndex.Name); if (forceUnlock) @@ -46,13 +58,8 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force } /// - protected override Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) + public Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) { - if (_taxonomyDirectoryFactory is null) - { - throw new NullReferenceException("Taxonomy Directory factory is null. Use constructor with all parameters"); - } - var dir = _taxonomyDirectoryFactory(luceneIndex.Name + "taxonomy"); if (forceUnlock) { diff --git a/src/Examine.Lucene/Directories/DirectoryFactoryBase.cs b/src/Examine.Lucene/Directories/DirectoryFactoryBase.cs deleted file mode 100644 index 4b093fd40..000000000 --- a/src/Examine.Lucene/Directories/DirectoryFactoryBase.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using System.Collections.Concurrent; -using Examine.Lucene.Providers; -using Directory = Lucene.Net.Store.Directory; - -namespace Examine.Lucene.Directories -{ - /// - public abstract class DirectoryFactoryBase : IDirectoryFactory - { - private readonly ConcurrentDictionary _createdDirectories = new ConcurrentDictionary(); - private bool _disposedValue; - - Directory IDirectoryFactory.CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) - => _createdDirectories.GetOrAdd( - luceneIndex.Name, - s => CreateDirectory(luceneIndex, forceUnlock)); - - Directory IDirectoryFactory.CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) - => _createdDirectories.GetOrAdd( - luceneIndex.Name + "_taxonomy", - s => CreateTaxonomyDirectory(luceneIndex, forceUnlock)); - - /// - protected abstract Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock); - - /// - protected virtual Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) => throw new NotSupportedException("Directory Factory does not implement CreateTaxonomyDirectory "); - - /// - /// Disposes the instance - /// - /// If the call is coming from the Dispose method - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - foreach (var d in _createdDirectories.Values) - { - d.Dispose(); - } - } - - _disposedValue = true; - } - } - - /// - /// Disposes this instance - /// - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method -#pragma warning disable IDE0022 // Use expression body for method - Dispose(disposing: true); -#pragma warning restore IDE0022 // Use expression body for method - } - } -} diff --git a/src/Examine.Lucene/Directories/FakeLuceneDirectoryIndexOptionsOptionsMonitor.cs b/src/Examine.Lucene/Directories/FakeLuceneDirectoryIndexOptionsOptionsMonitor.cs new file mode 100644 index 000000000..873e63c2b --- /dev/null +++ b/src/Examine.Lucene/Directories/FakeLuceneDirectoryIndexOptionsOptionsMonitor.cs @@ -0,0 +1,16 @@ +using System; +using Microsoft.Extensions.Options; + +namespace Examine.Lucene.Directories +{ + internal sealed class FakeLuceneDirectoryIndexOptionsOptionsMonitor : IOptionsMonitor + { + private static readonly LuceneDirectoryIndexOptions Default = new LuceneDirectoryIndexOptions(); + + public LuceneDirectoryIndexOptions CurrentValue => Default; + + public LuceneDirectoryIndexOptions Get(string? name) => Default; + + public IDisposable OnChange(Action listener) => throw new NotImplementedException(); + } +} diff --git a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs index 8c1ea5bcb..2b34ef9df 100644 --- a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs @@ -1,7 +1,9 @@ +using System; using System.IO; using Examine.Lucene.Providers; using Lucene.Net.Index; using Lucene.Net.Store; +using Microsoft.Extensions.Options; using Directory = Lucene.Net.Store.Directory; namespace Examine.Lucene.Directories @@ -9,19 +11,21 @@ namespace Examine.Lucene.Directories /// /// Represents a directory factory for creating file system directories /// - public class FileSystemDirectoryFactory : DirectoryFactoryBase + public class FileSystemDirectoryFactory : IDirectoryFactory { private readonly DirectoryInfo _baseDir; /// /// Creates an instance of /// - /// The base directory - /// The lock factory - public FileSystemDirectoryFactory(DirectoryInfo baseDir, ILockFactory lockFactory) + public FileSystemDirectoryFactory( + DirectoryInfo baseDir, + ILockFactory lockFactory, + IOptionsMonitor indexOptions) { _baseDir = baseDir; LockFactory = lockFactory; + IndexOptions = indexOptions; } /// @@ -29,8 +33,13 @@ public FileSystemDirectoryFactory(DirectoryInfo baseDir, ILockFactory lockFactor /// public ILockFactory LockFactory { get; } + /// + /// Provides access to index options for Lucene directories. + /// + protected IOptionsMonitor IndexOptions { get; } + /// - protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) + public virtual Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) { var path = Path.Combine(_baseDir.FullName, luceneIndex.Name); var luceneIndexFolder = new DirectoryInfo(path); @@ -40,13 +49,22 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force { IndexWriter.Unlock(dir); } - return dir; + + var options = IndexOptions.GetNamedOptions(luceneIndex.Name); + if (options.NrtEnabled) + { + return new NRTCachingDirectory(dir, options.NrtCacheMaxMergeSizeMB, options.NrtCacheMaxCachedMB); + } + else + { + return dir; + } } /// - protected override Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) + public Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock) { - var path = Path.Combine(_baseDir.FullName, luceneIndex.Name,"taxonomy"); + var path = Path.Combine(_baseDir.FullName, luceneIndex.Name, "taxonomy"); var luceneIndexFolder = new DirectoryInfo(path); var dir = FSDirectory.Open(luceneIndexFolder, LockFactory.GetLockFactory(luceneIndexFolder)); @@ -54,7 +72,15 @@ protected override Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bo { IndexWriter.Unlock(dir); } - return dir; + var options = IndexOptions.GetNamedOptions(luceneIndex.Name); + if (options.NrtEnabled) + { + return new NRTCachingDirectory(dir, options.NrtCacheMaxMergeSizeMB, options.NrtCacheMaxCachedMB); + } + else + { + return dir; + } } } } diff --git a/src/Examine.Lucene/Directories/IDirectoryFactory.cs b/src/Examine.Lucene/Directories/IDirectoryFactory.cs index c59228b10..4afb5f60e 100644 --- a/src/Examine.Lucene/Directories/IDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/IDirectoryFactory.cs @@ -8,9 +8,9 @@ namespace Examine.Lucene.Directories /// Creates a Lucene for an index /// /// - /// The directory created must only be created ONCE per index and disposed when the factory is disposed. + /// Used by the index to create directory instance for the index. The index is responsible for managing the lifetime of the directory instance. /// - public interface IDirectoryFactory : IDisposable + public interface IDirectoryFactory { /// /// Creates the directory instance @@ -21,7 +21,7 @@ public interface IDirectoryFactory : IDisposable /// /// Any subsequent calls for the same index will return the same directory instance /// - Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock); + public Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock); /// /// Creates the directory instance for the Taxonomy Index @@ -32,6 +32,6 @@ public interface IDirectoryFactory : IDisposable /// /// Any subsequent calls for the same index will return the same directory instance /// - Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock); + public Directory CreateTaxonomyDirectory(LuceneIndex luceneIndex, bool forceUnlock); } } diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectory.cs new file mode 100644 index 000000000..a907358b5 --- /dev/null +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectory.cs @@ -0,0 +1,48 @@ +using System.IO; +using Examine.Lucene.Providers; +using Lucene.Net.Store; +using Microsoft.Extensions.Logging; +using Directory = Lucene.Net.Store.Directory; + +namespace Examine.Lucene.Directories +{ + internal sealed class SyncedFileSystemDirectory : FilterDirectory + { + private readonly ExamineReplicator _replicator; + + public SyncedFileSystemDirectory( + ILogger replicatorLogger, + ILogger clientLogger, + Directory localLuceneDirectory, + Directory mainLuceneDirectory, + Directory mainTaxonomyLuceneDirectory, + LuceneIndex luceneIndex, + DirectoryInfo tempDir) + : base(localLuceneDirectory) + { + // now create the replicator that will copy from local to main on schedule + _replicator = new ExamineReplicator(replicatorLogger, clientLogger, luceneIndex, localLuceneDirectory, mainLuceneDirectory, mainTaxonomyLuceneDirectory, tempDir); + LocalLuceneDirectory = localLuceneDirectory; + MainLuceneDirectory = mainLuceneDirectory; + } + + internal Directory LocalLuceneDirectory { get; } + + internal Directory MainLuceneDirectory { get; } + + public override Lock MakeLock(string name) + { + // Start replicating back to main, this is ok to call multiple times, it will only execute once. + _replicator.StartIndexReplicationOnSchedule(1000); + + return base.MakeLock(name); + } + + protected override void Dispose(bool disposing) + { + _replicator.Dispose(); + MainLuceneDirectory.Dispose(); + base.Dispose(disposing); + } + } +} diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index 012baf161..05de53fcf 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -1,10 +1,10 @@ - using System; using System.IO; -using System.Threading; using Examine.Lucene.Providers; using Lucene.Net.Analysis.Standard; +using Lucene.Net.Facet.Taxonomy.Directory; using Lucene.Net.Index; +using Lucene.Net.Replicator; using Lucene.Net.Store; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -14,102 +14,405 @@ namespace Examine.Lucene.Directories { /// /// A directory factory that replicates the index from main storage on initialization to another - /// directory, then creates a lucene Directory based on that replicated index. A replication thread - /// is spawned to then replicate the local index back to the main storage location. + /// directory, then creates a Lucene Directory based on that replicated index. /// /// + /// A replication thread is spawned to then replicate the local index back to the main storage location. /// By default, Examine configures the local directory to be the %temp% folder. + /// This also checks if the main/local storage indexes are healthy and syncs/removes accordingly. /// public class SyncedFileSystemDirectoryFactory : FileSystemDirectoryFactory { private readonly DirectoryInfo _localDir; + private readonly DirectoryInfo _mainDir; private readonly ILoggerFactory _loggerFactory; - private ExamineReplicator? _replicator; + private readonly bool _tryFixMainIndexIfCorrupt; + private readonly ILogger _logger; + private readonly ILogger _replicatorLogger; + private readonly ILogger _clientLogger; - /// + /// + /// Initializes a new instance of the class. + /// + /// The local directory where the index will be replicated. + /// The main directory where the index is stored. + /// The lock factory used for managing index locks. + /// The logger factory used for creating loggers. + /// The options monitor for Lucene directory index options. public SyncedFileSystemDirectoryFactory( DirectoryInfo localDir, DirectoryInfo mainDir, ILockFactory lockFactory, - ILoggerFactory loggerFactory) - : base(mainDir, lockFactory) + ILoggerFactory loggerFactory, + IOptionsMonitor indexOptions) + : this(localDir, mainDir, lockFactory, loggerFactory, indexOptions, false) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The local directory where the index will be replicated. + /// The main directory where the index is stored. + /// The lock factory used for managing index locks. + /// The logger factory used for creating loggers. + /// The options monitor for Lucene directory index options. + /// Indicates whether to attempt fixing the main index if it is corrupt. + public SyncedFileSystemDirectoryFactory( + DirectoryInfo localDir, + DirectoryInfo mainDir, + ILockFactory lockFactory, + ILoggerFactory loggerFactory, + IOptionsMonitor indexOptions, + bool tryFixMainIndexIfCorrupt) + : base(mainDir, lockFactory, indexOptions) { _localDir = localDir; + _mainDir = mainDir; _loggerFactory = loggerFactory; + _tryFixMainIndexIfCorrupt = tryFixMainIndexIfCorrupt; + _logger = _loggerFactory.CreateLogger(); + _replicatorLogger = _loggerFactory.CreateLogger(); + _clientLogger = _loggerFactory.CreateLogger(); } - /// - protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) + internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlock, out Directory directory) { - var path = Path.Combine(_localDir.FullName, luceneIndex.Name); - var localLuceneIndexFolder = new DirectoryInfo(path); + var mainPath = Path.Combine(_mainDir.FullName, luceneIndex.Name); + var mainLuceneIndexFolder = new DirectoryInfo(mainPath); + var mainPathTaxonomyPath = Path.Combine(_mainDir.FullName, luceneIndex.Name, "taxonomy"); + var mainLuceneTaxonomyIndexFolder = new DirectoryInfo(mainPathTaxonomyPath); - var mainDir = base.CreateDirectory(luceneIndex, forceUnlock); + var localPath = Path.Combine(_localDir.FullName, luceneIndex.Name); + var localLuceneIndexFolder = new DirectoryInfo(localPath); + var localTaxonomyPath = Path.Combine(_localDir.FullName, luceneIndex.Name, "taxonomy"); + var localLuceneTaxonomyIndexFolder = new DirectoryInfo(localTaxonomyPath); // used by the replicator, will be a short lived directory for each synced revision and deleted when finished. var tempDir = new DirectoryInfo(Path.Combine(_localDir.FullName, "Rep", Guid.NewGuid().ToString("N"))); - if (DirectoryReader.IndexExists(mainDir)) + var mainLuceneDir = base.CreateDirectory(luceneIndex, forceUnlock); + var mainTaxonomyDir = base.CreateTaxonomyDirectory(luceneIndex, forceUnlock); + var localLuceneDir = FSDirectory.Open( + localLuceneIndexFolder, + LockFactory.GetLockFactory(localLuceneIndexFolder)); + var localLuceneTaxonomyDir = FSDirectory.Open( + localLuceneTaxonomyIndexFolder, + LockFactory.GetLockFactory(localLuceneTaxonomyIndexFolder)); + + var mainIndexExists = DirectoryReader.IndexExists(mainLuceneDir); + var localIndexExists = DirectoryReader.IndexExists(localLuceneDir); + var mainTaxonomyIndexExists = DirectoryReader.IndexExists(mainTaxonomyDir); + var localTaxonomyIndexExists = DirectoryReader.IndexExists(localLuceneTaxonomyDir); + + // Both must exist for the main index to be considered healthy + var hasMainIndexes = mainIndexExists && mainTaxonomyIndexExists; + var hasLocalIndexes = localIndexExists && localTaxonomyIndexExists; + + var mainResult = CreateResult.Init; + + if (hasMainIndexes) + { + mainResult = CheckIndexHealthAndFix(mainLuceneDir, mainLuceneIndexFolder, luceneIndex.Name, _tryFixMainIndexIfCorrupt); + mainResult |= CheckIndexHealthAndFix(mainTaxonomyDir, mainLuceneTaxonomyIndexFolder, $"{luceneIndex.Name}.taxonomy", _tryFixMainIndexIfCorrupt); + } + + // the main index is/was unhealthy or missing, lets check the local index if it exists + if (hasLocalIndexes && (!hasMainIndexes || mainResult.HasFlag(CreateResult.NotClean) || mainResult.HasFlag(CreateResult.MissingSegments))) + { + // TODO: add details here and more below too + + var localResult = CheckIndexHealthAndFix(localLuceneDir, localLuceneIndexFolder, luceneIndex.Name, false); + localResult |= CheckIndexHealthAndFix(localLuceneTaxonomyDir, localLuceneTaxonomyIndexFolder, $"{luceneIndex.Name}.taxonomy", false); + + if (localResult == CreateResult.Init) + { + // it was read successfully, we can sync back to main + localResult |= TryGetIndexWriters(OpenMode.APPEND, localLuceneDir, localLuceneIndexFolder, localLuceneTaxonomyDir, localLuceneTaxonomyIndexFolder, false, luceneIndex.Name, out var indexWriter, out var taxonomyWriterFactory); + if (localResult.HasFlag(CreateResult.OpenedSuccessfully)) + { + using (indexWriter!) + using (taxonomyWriterFactory!.IndexWriter!) + { + SyncIndex(indexWriter!, taxonomyWriterFactory!, true, luceneIndex.Name, mainLuceneIndexFolder, mainLuceneTaxonomyIndexFolder, tempDir); + mainResult |= CreateResult.SyncedFromLocal; + // we need to check the main index again, as it may have been fixed by the sync + mainIndexExists = DirectoryReader.IndexExists(mainLuceneDir); + mainTaxonomyIndexExists = DirectoryReader.IndexExists(mainTaxonomyDir); + hasMainIndexes = mainIndexExists && mainTaxonomyIndexExists; + } + } + } + } + + if (hasMainIndexes) { // when the lucene directory is going to be created, we'll sync from main storage to local // storage before any index/writer is opened. - using (var tempMainIndexWriter = new IndexWriter( - mainDir, - new IndexWriterConfig( - LuceneInfo.CurrentVersion, - new StandardAnalyzer(LuceneInfo.CurrentVersion)) - { - OpenMode = OpenMode.APPEND, - IndexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()) - })) - using (var tempMainIndex = new LuceneIndex(_loggerFactory, luceneIndex.Name, new TempOptions(), tempMainIndexWriter)) - using (var tempLocalDirectory = new SimpleFSDirectory(localLuceneIndexFolder, LockFactory.GetLockFactory(localLuceneIndexFolder))) - using (var replicator = new ExamineReplicator(_loggerFactory, tempMainIndex, tempLocalDirectory, tempDir)) + + var openMode = mainResult == CreateResult.Init || mainResult.HasFlag(CreateResult.Fixed) || mainResult.HasFlag(CreateResult.SyncedFromLocal) + ? OpenMode.APPEND + : OpenMode.CREATE; + + mainResult |= TryGetIndexWriters(openMode, mainLuceneDir, mainLuceneIndexFolder, mainTaxonomyDir, mainLuceneTaxonomyIndexFolder, true, luceneIndex.Name, out var indexWriter, out var taxonomyWriterFactory); + if (indexWriter is not null && taxonomyWriterFactory is not null) { - if (forceUnlock) + using (indexWriter) + using (taxonomyWriterFactory!.IndexWriter) { - IndexWriter.Unlock(tempLocalDirectory); + if (!mainResult.HasFlag(CreateResult.SyncedFromLocal)) + { + SyncIndex(indexWriter, taxonomyWriterFactory, forceUnlock, luceneIndex.Name, localLuceneIndexFolder, localLuceneTaxonomyIndexFolder, tempDir); + } } - - // replicate locally. - replicator.ReplicateIndex(); } } - // now create the replicator that will copy from local to main on schedule - _replicator = new ExamineReplicator(_loggerFactory, luceneIndex, mainDir, tempDir); - var localLuceneDir = FSDirectory.Open( - localLuceneIndexFolder, - LockFactory.GetLockFactory(localLuceneIndexFolder)); - if (forceUnlock) { IndexWriter.Unlock(localLuceneDir); + IndexWriter.Unlock(localLuceneTaxonomyDir); + } + + Directory activeLocalLuceneDir; + + var options = IndexOptions.GetNamedOptions(luceneIndex.Name); + if (options.NrtEnabled) + { + activeLocalLuceneDir = new NRTCachingDirectory(localLuceneDir, options.NrtCacheMaxMergeSizeMB, options.NrtCacheMaxCachedMB); + } + else + { + activeLocalLuceneDir = localLuceneDir; } - // Start replicating back to main - _replicator.StartIndexReplicationOnSchedule(1000); + directory = new SyncedFileSystemDirectory(_replicatorLogger, _clientLogger, activeLocalLuceneDir, mainLuceneDir, mainTaxonomyDir, luceneIndex, tempDir); - return localLuceneDir; + return mainResult; } - /// - /// Disposes the instance - /// - /// If the call is coming from Dispose - protected override void Dispose(bool disposing) + [Flags] + internal enum CreateResult + { + Init = 0, + MissingSegments = 1, + NotClean = 2, + Fixed = 4, + NotFixed = 8, + ExceptionNotFixed = 16, + CorruptCreatedNew = 32, + OpenedSuccessfully = 64, + SyncedFromLocal = 128 + } + + /// + public override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) + { + _ = TryCreateDirectory(luceneIndex, forceUnlock, out var directory); + return directory; + } + + private CreateResult TryGetIndexWriters( + OpenMode openMode, + Directory luceneDirectory, + DirectoryInfo directoryInfo, + Directory taxonomyDirectory, + DirectoryInfo taxonomyDirectoryInfo, + bool createNewIfCorrupt, + string indexName, + out IndexWriter? indexWriter, + out SnapshotDirectoryTaxonomyIndexWriterFactory? snapshotDirectoryTaxonomyIndexWriterFactory) + { + try + { + indexWriter = GetIndexWriter(luceneDirectory, openMode); + var directoryTaxonomyWriter = GetTaxonomyWriter(taxonomyDirectory, openMode, out snapshotDirectoryTaxonomyIndexWriterFactory); + + if (openMode == OpenMode.APPEND) + { + return CreateResult.OpenedSuccessfully; + } + else + { + // Required to remove old index files which can be problematic + // if they remain in the index folder when replication is attempted. + indexWriter.Commit(); + indexWriter.WaitForMerges(); + directoryTaxonomyWriter.Commit(); + + return CreateResult.CorruptCreatedNew; + } + } + catch (Exception ex) + { + if (createNewIfCorrupt) + { + // Index is corrupted, typically this will be FileNotFoundException or CorruptIndexException + _logger.LogError(ex, "{IndexName} at {IndexPath} index is corrupt, a new one will be created", indexName, directoryInfo.FullName); + + // Totally clear all files in the directory + ClearDirectory(directoryInfo); + ClearDirectory(taxonomyDirectoryInfo); + + indexWriter = GetIndexWriter(luceneDirectory, OpenMode.CREATE); + _ = GetTaxonomyWriter(taxonomyDirectory, OpenMode.CREATE, out snapshotDirectoryTaxonomyIndexWriterFactory); + } + else + { + indexWriter = null; + snapshotDirectoryTaxonomyIndexWriterFactory = null; + } + + return CreateResult.CorruptCreatedNew; + } + } + + private void ClearDirectory(DirectoryInfo directoryInfo) + { + if (directoryInfo.Exists) + { + foreach (var file in directoryInfo.EnumerateFiles()) + { + file.Delete(); + } + } + } + + private void SyncIndex( + IndexWriter sourceIndexWriter, + SnapshotDirectoryTaxonomyIndexWriterFactory sourceTaxonomyWriterFactory, + bool forceUnlock, + string indexName, + DirectoryInfo destinationDirectory, + DirectoryInfo destinationTaxonomyDirectory, + DirectoryInfo tempDir) + { + // First, we need to clear the main index. If for some reason it is at the same revision, the syncing won't do anything. + ClearDirectory(destinationDirectory); + ClearDirectory(destinationTaxonomyDirectory); + + using (var sourceIndex = new LuceneIndex(_loggerFactory, indexName, new TempOptions(), sourceIndexWriter, sourceTaxonomyWriterFactory)) + using (var destinationLuceneDirectory = FSDirectory.Open(destinationDirectory, LockFactory.GetLockFactory(destinationDirectory))) + using (var destinationLuceneTaxonomyDirectory = FSDirectory.Open(destinationDirectory, LockFactory.GetLockFactory(destinationTaxonomyDirectory))) + using (var replicator = new ExamineReplicator( + _replicatorLogger, + _clientLogger, + sourceIndex, + sourceIndexWriter.Directory, + destinationLuceneDirectory, + destinationLuceneTaxonomyDirectory, + tempDir)) + { + if (forceUnlock) + { + IndexWriter.Unlock(destinationLuceneDirectory); + } + + // replicate locally. + replicator.ReplicateIndex(); + } + } + + private CreateResult CheckIndexHealthAndFix( + Directory luceneDir, + DirectoryInfo directoryInfo, + string indexName, + bool doFix) + { + using var writer = new StringWriter(); + var result = CreateResult.Init; + + var checker = new CheckIndex(luceneDir) + { + // Redirect the logging output of the checker + InfoStream = writer + }; + + var status = checker.DoCheckIndex(); + writer.Flush(); + + _logger.LogDebug("{IndexName} health check report {IndexReport}", indexName, writer.ToString()); + + if (status.MissingSegments) + { + _logger.LogWarning("{IndexName} index at {IndexPath} is missing segments, it will be deleted.", indexName, directoryInfo.FullName); + result = CreateResult.MissingSegments; + } + else if (!status.Clean) + { + _logger.LogWarning("Checked index {IndexName} at {IndexPath} and it is not clean.", indexName, directoryInfo.FullName); + result = CreateResult.NotClean; + + if (doFix) + { + _logger.LogWarning("Attempting to fix {IndexName} at {IndexPath}. {DocumentsLost} documents will be lost.", indexName, status.TotLoseDocCount, directoryInfo.FullName); + + try + { + checker.FixIndex(status); + status = checker.DoCheckIndex(); + + if (!status.Clean) + { + _logger.LogError("{IndexName} index at {IndexPath} could not be fixed, it will be deleted.", indexName, directoryInfo.FullName); + result |= CreateResult.NotFixed; + } + else + { + _logger.LogInformation("Index {IndexName} at {IndexPath} fixed. {DocumentsLost} documents were lost.", indexName, status.TotLoseDocCount, directoryInfo.FullName); + result |= CreateResult.Fixed; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "{IndexName} index at {IndexPath} could not be fixed, it will be deleted.", indexName, directoryInfo.FullName); + result |= CreateResult.ExceptionNotFixed; + } + } + } + else + { + _logger.LogInformation("Checked index {IndexName} at {IndexPath} and it is clean.", indexName, directoryInfo.FullName); + } + + return result; + } + + private static IndexWriter GetIndexWriter(Directory mainDir, OpenMode openMode) { - base.Dispose(disposing); - if (disposing) + var indexWriter = new IndexWriter( + mainDir, + new IndexWriterConfig( + LuceneInfo.CurrentVersion, + new StandardAnalyzer(LuceneInfo.CurrentVersion)) + { + OpenMode = openMode, + IndexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), + MergePolicy = new TieredMergePolicy() + }); + + return indexWriter; + } + + private static DirectoryTaxonomyWriter GetTaxonomyWriter(Directory d, OpenMode openMode, out SnapshotDirectoryTaxonomyIndexWriterFactory snapshotDirectoryTaxonomyIndexWriterFactory) + { + if (d == null) { - _replicator?.Dispose(); + ArgumentNullException.ThrowIfNull(nameof(d)); } + + // TODO: This API is broken and weird, the SnapshotDirectoryTaxonomyIndexWriterFactory hangs on the to the writer that it creates, + // and is needed downstream by the replicator APIs. + snapshotDirectoryTaxonomyIndexWriterFactory = new SnapshotDirectoryTaxonomyIndexWriterFactory(); + var writer = new DirectoryTaxonomyWriter(snapshotDirectoryTaxonomyIndexWriterFactory, d, openMode, DirectoryTaxonomyWriter.CreateDefaultTaxonomyWriterCache()); + return writer; } private class TempOptions : IOptionsMonitor { public LuceneDirectoryIndexOptions CurrentValue => new LuceneDirectoryIndexOptions(); - public LuceneDirectoryIndexOptions Get(string name) => CurrentValue; + + public LuceneDirectoryIndexOptions Get(string? name) => CurrentValue; public IDisposable OnChange(Action listener) => throw new NotImplementedException(); } diff --git a/src/Examine.Lucene/Directories/SyncedTaxonomyFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedTaxonomyFileSystemDirectoryFactory.cs deleted file mode 100644 index b94eb0f31..000000000 --- a/src/Examine.Lucene/Directories/SyncedTaxonomyFileSystemDirectoryFactory.cs +++ /dev/null @@ -1,125 +0,0 @@ - -using System; -using System.IO; -using System.Threading; -using Examine.Lucene.Providers; -using Lucene.Net.Analysis.Standard; -using Lucene.Net.Index; -using Lucene.Net.Store; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Directory = Lucene.Net.Store.Directory; - -namespace Examine.Lucene.Directories -{ - /// - /// A directory factory that replicates the index and it's Taxonomy Index from main storage on initialization to another - /// directory, then creates a lucene Directory based on that replicated index. A replication thread - /// is spawned to then replicate the local index back to the main storage location. - /// - /// - /// By default, Examine configures the local directory to be the %temp% folder. - /// - public class SyncedTaxonomyFileSystemDirectoryFactory : FileSystemDirectoryFactory - { - private readonly DirectoryInfo _localDir; - private readonly ILoggerFactory _loggerFactory; - private ExamineTaxonomyReplicator? _replicator; - - /// - public SyncedTaxonomyFileSystemDirectoryFactory( - DirectoryInfo localDir, - DirectoryInfo mainDir, - ILockFactory lockFactory, - ILoggerFactory loggerFactory) - : base(mainDir, lockFactory) - { - _localDir = localDir; - _loggerFactory = loggerFactory; - } - - /// - protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) - { - var luceneTaxonomyIndex = luceneIndex as LuceneIndex; - var path = Path.Combine(_localDir.FullName, luceneIndex.Name); - var localLuceneIndexFolder = new DirectoryInfo(path); - - Directory mainDir = base.CreateDirectory(luceneIndex, forceUnlock); - - var taxonomyPath = Path.Combine(_localDir.FullName, luceneIndex.Name, "taxonomy"); - var localLuceneTaxonomyIndexFolder = new DirectoryInfo(taxonomyPath); - - Directory mainTaxonomyDir = base.CreateTaxonomyDirectory(luceneTaxonomyIndex, forceUnlock); - - // used by the replicator, will be a short lived directory for each synced revision and deleted when finished. - var tempDir = new DirectoryInfo(Path.Combine(_localDir.FullName, "Rep", Guid.NewGuid().ToString("N"))); - - if (DirectoryReader.IndexExists(mainDir) && DirectoryReader.IndexExists(mainTaxonomyDir)) - { - // when the lucene directory is going to be created, we'll sync from main storage to local - // storage before any index/writer is opened. - using (var tempMainIndexWriter = new IndexWriter( - mainDir, - new IndexWriterConfig( - LuceneInfo.CurrentVersion, - new StandardAnalyzer(LuceneInfo.CurrentVersion)) - { - OpenMode = OpenMode.APPEND, - IndexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()) - })) - using (var tempMainIndex = new LuceneIndex(_loggerFactory, luceneIndex.Name, new TempOptions(), tempMainIndexWriter)) - using (var tempLocalDirectory = new SimpleFSDirectory(localLuceneIndexFolder, LockFactory.GetLockFactory(localLuceneIndexFolder))) - using (var tempTaxonomyLocalDirectory = new SimpleFSDirectory(localLuceneTaxonomyIndexFolder, LockFactory.GetLockFactory(localLuceneTaxonomyIndexFolder))) - using (var replicator = new ExamineTaxonomyReplicator(_loggerFactory, tempMainIndex, tempLocalDirectory, tempTaxonomyLocalDirectory, tempDir)) - { - if (forceUnlock) - { - IndexWriter.Unlock(tempLocalDirectory); - } - - // replicate locally. - replicator.ReplicateIndex(); - } - } - - // now create the replicator that will copy from local to main on schedule - _replicator = new ExamineTaxonomyReplicator(_loggerFactory, luceneTaxonomyIndex, mainDir, mainTaxonomyDir, tempDir); - var localLuceneDir = FSDirectory.Open( - localLuceneIndexFolder, - LockFactory.GetLockFactory(localLuceneIndexFolder)); - - if (forceUnlock) - { - IndexWriter.Unlock(localLuceneDir); - } - - // Start replicating back to main - _replicator.StartIndexReplicationOnSchedule(1000); - - return localLuceneDir; - } - - /// - /// Disposes the instance - /// - /// If the call is coming from Dispose - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - if (disposing) - { - _replicator?.Dispose(); - } - } - - private class TempOptions : IOptionsMonitor - { - public LuceneDirectoryIndexOptions CurrentValue => new LuceneDirectoryIndexOptions(); - public LuceneDirectoryIndexOptions Get(string name) => CurrentValue; - - public IDisposable OnChange(Action listener) => throw new NotImplementedException(); - } - - } -} diff --git a/src/Examine.Lucene/Directories/TempEnvFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/TempEnvFileSystemDirectoryFactory.cs index 9d0d420d1..e1d1fc9b4 100644 --- a/src/Examine.Lucene/Directories/TempEnvFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/TempEnvFileSystemDirectoryFactory.cs @@ -1,30 +1,36 @@ using System; using System.IO; +using Microsoft.Extensions.Options; namespace Examine.Lucene.Directories { - /// - /// A directory factory used to create an instance of FSDirectory that uses the current %temp% environment variable + /// A directory factory used to create an instance of FSDirectory that uses the current %temp% environment variable. /// /// - /// This works well for Azure Web Apps directory sync + /// This works well for Azure Web Apps directory sync. /// public class TempEnvFileSystemDirectoryFactory : FileSystemDirectoryFactory { - /// + /// + /// Initializes a new instance of the class. + /// + /// The application identifier used to generate a unique path. + /// The lock factory used for directory locking. + /// The options monitor for Lucene directory index options. public TempEnvFileSystemDirectoryFactory( IApplicationIdentifier applicationIdentifier, - ILockFactory lockFactory) - : base(new DirectoryInfo(GetTempPath(applicationIdentifier)), lockFactory) + ILockFactory lockFactory, + IOptionsMonitor indexOptions) + : base(new DirectoryInfo(GetTempPath(applicationIdentifier)), lockFactory, indexOptions) { } /// - /// Gets a temp path for examine indexes + /// Gets a temp path for examine indexes. /// - /// - /// + /// The application identifier used to generate a unique path. + /// The temporary path for examine indexes. public static string GetTempPath(IApplicationIdentifier applicationIdentifier) { var appDomainHash = applicationIdentifier.GetApplicationUniqueIdentifier().GenerateHash(); @@ -32,9 +38,9 @@ public static string GetTempPath(IApplicationIdentifier applicationIdentifier) var cachePath = Path.Combine( Path.GetTempPath(), "ExamineIndexes", - //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back - // to worker A again, in theory the %temp% folder should already be empty but we really want to make sure that its not - // utilizing an old index + // Include the appdomain hash as a safety check, for example if a website is moved from worker A to worker B and then back + // to worker A again, in theory the %temp% folder should already be empty but we really want to make sure that it's not + // utilizing an old index. appDomainHash); return cachePath; diff --git a/src/Examine.Lucene/Examine.Lucene.csproj b/src/Examine.Lucene/Examine.Lucene.csproj index 0ddf6cadd..7fe8b6a52 100644 --- a/src/Examine.Lucene/Examine.Lucene.csproj +++ b/src/Examine.Lucene/Examine.Lucene.csproj @@ -21,15 +21,18 @@ <_Parameter1>Examine.Test + + <_Parameter1>Examine.Benchmarks + - 4.8.0-beta00016 + 4.8.0-beta00017 - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -38,7 +41,7 @@ 4.3.0 - 6.0.0 + 8.0.0 diff --git a/src/Examine.Lucene/ExamineReplicator.cs b/src/Examine.Lucene/ExamineReplicator.cs index 627abd60e..72eab9824 100644 --- a/src/Examine.Lucene/ExamineReplicator.cs +++ b/src/Examine.Lucene/ExamineReplicator.cs @@ -1,9 +1,11 @@ using System; using System.IO; +using Examine.Lucene.Directories; using Examine.Lucene.Providers; using Lucene.Net.Index; using Lucene.Net.Replicator; using Lucene.Net.Store; +using Lucene.Net.Util; using Microsoft.Extensions.Logging; using Directory = Lucene.Net.Store.Directory; @@ -18,57 +20,68 @@ namespace Examine.Lucene public class ExamineReplicator : IDisposable { private bool _disposedValue; - private readonly IReplicator _replicator; + private readonly LocalReplicator _replicator; private readonly LuceneIndex _sourceIndex; + private readonly Directory _sourceDirectory; private readonly Directory _destinationDirectory; - private readonly ReplicationClient _localReplicationClient; + private readonly Lazy _localReplicationClient; private readonly object _locker = new object(); private bool _started = false; private readonly ILogger _logger; /// - /// Creates an instance of + /// Initializes a new instance of the class. /// - /// The logger factory - /// The source index - /// The destination directory - /// The temp storage directory info + /// The logger for the replicator. + /// The logger for the replication client. + /// The source index to replicate from. + /// The source directory of the index. + /// The destination directory for replication. + /// + /// The temporary storage directory used during replication. public ExamineReplicator( - ILoggerFactory loggerFactory, + ILogger replicatorLogger, + ILogger clientLogger, LuceneIndex sourceIndex, + Directory sourceDirectory, Directory destinationDirectory, + Directory destinationTaxonomyDirectory, DirectoryInfo tempStorage) { _sourceIndex = sourceIndex; + _sourceDirectory = sourceDirectory; _destinationDirectory = destinationDirectory; _replicator = new LocalReplicator(); - _logger = loggerFactory.CreateLogger(); - - _localReplicationClient = new LoggingReplicationClient( - loggerFactory.CreateLogger(), - _replicator, - new IndexReplicationHandler( - destinationDirectory, - () => - { - if (_logger.IsEnabled(LogLevel.Debug)) + _logger = replicatorLogger; + + _localReplicationClient = new Lazy(() + => new LoggingReplicationClient( + clientLogger, + _replicator, + new IndexAndTaxonomyReplicationHandler( + destinationDirectory, + destinationTaxonomyDirectory, + () => { - var sourceDir = sourceIndex.GetLuceneDirectory() as FSDirectory; - var destDir = destinationDirectory as FSDirectory; - - // Callback, can be used to notifiy when replication is done (i.e. to open the index) + // Callback, can be used to notify when replication is done (i.e. to open the index) if (_logger.IsEnabled(LogLevel.Debug)) { + var sourceDir = UnwrapDirectory(sourceDirectory); + var destDir = UnwrapDirectory(destinationDirectory); + var sourceTaxonomyDir = sourceIndex.GetLuceneTaxonomyDirectory() as FSDirectory; + var destTaxonomyDir = destinationTaxonomyDirectory as FSDirectory; + _logger.LogDebug( - "{IndexName} replication complete from {SourceDirectory} to {DestinationDirectory}", + "{IndexName} replication complete from {SourceDirectory} to {DestinationDirectory} and Taxonomy {TaxonomySourceDirectory} to {TaxonomyDestinationDirectory}", sourceIndex.Name, sourceDir?.Directory.ToString() ?? "InMemory", - destDir?.Directory.ToString() ?? "InMemory"); + destDir?.Directory.ToString() ?? "InMemory", + sourceTaxonomyDir?.Directory.ToString() ?? "InMemory", + destTaxonomyDir?.Directory.ToString() ?? "InMemory" + ); } - } - - }), - new PerSessionDirectoryFactory(tempStorage.FullName)); + }), + new PerSessionDirectoryFactory(tempStorage.FullName))); } /// @@ -81,19 +94,30 @@ public void ReplicateIndex() throw new InvalidOperationException("The destination directory is locked"); } - IndexRevision rev; + _logger.LogInformation( + "Replicating index from {SourceIndex} to {DestinationIndex}", + _sourceDirectory, + _destinationDirectory); + + IndexAndTaxonomyRevision rev; try { - rev = new IndexRevision(_sourceIndex.IndexWriter.IndexWriter); + rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.SnapshotDirectoryTaxonomyIndexWriterFactory); } catch (InvalidOperationException) { // will occur if there is nothing to sync + _logger.LogInformation("There was nothing to replicate to {DestinationIndex}", _destinationDirectory); return; } _replicator.Publish(rev); - _localReplicationClient.UpdateNow(); + _localReplicationClient.Value.UpdateNow(); + + _logger.LogInformation( + "Replication from index {SourceIndex} to {DestinationIndex} complete.", + _sourceDirectory, + _destinationDirectory); } /// @@ -103,15 +127,20 @@ public void ReplicateIndex() /// public void StartIndexReplicationOnSchedule(int milliseconds) { + if (_started) + { + return; + } + lock (_locker) { - if (_started) + _started = true; + + if (_sourceIndex.IsCancellationRequested) { return; } - _started = true; - if (IndexWriter.IsLocked(_destinationDirectory)) { throw new InvalidOperationException("The destination directory is locked"); @@ -121,7 +150,7 @@ public void StartIndexReplicationOnSchedule(int milliseconds) // this will update the destination every second if there are changes. // the change monitor will be stopped when this is disposed. - _localReplicationClient.StartUpdateThread(milliseconds, $"IndexRep{_sourceIndex.Name}"); + _localReplicationClient.Value.StartUpdateThread(milliseconds, $"IndexRep{_sourceIndex.Name}"); } } @@ -142,8 +171,12 @@ private void SourceIndex_IndexCommitted(object? sender, EventArgs e) } _logger.LogDebug("{IndexName} committed", index?.Name ?? $"({nameof(index)} is null)"); } - var rev = new IndexRevision(_sourceIndex.IndexWriter.IndexWriter); - _replicator.Publish(rev); + + if (!_sourceIndex.IsCancellationRequested) + { + var rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.SnapshotDirectoryTaxonomyIndexWriterFactory); + _replicator.Publish(rev); + } } /// @@ -157,22 +190,62 @@ protected virtual void Dispose(bool disposing) if (disposing) { _sourceIndex.IndexCommitted -= SourceIndex_IndexCommitted; - _localReplicationClient.Dispose(); + + // Disposal in this order based on lucene.net tests: + // https://github.com/apache/lucenenet/blob/6b161d961a7764f2d2dbe90ee2ae03f73ccce019/src/Lucene.Net.Tests.Replicator/IndexReplicationClientTest.cs#L169 + // replicator client + // writer + // replicator + // publish directory + // handler directory + + // We have: + // writer - done with LuceneIndex + // SyncedFileSystemDirectory - done with LuceneIndex + // - ExamineReplicator (this) + // -- client + // --- replicator + // - publish directory + // - handler directory - done with base class FilterDirectory + if (_localReplicationClient.IsValueCreated) + { + _localReplicationClient.Value.Dispose(); + } + _replicator.Dispose(); } _disposedValue = true; } } - /// - /// Disposes the instance - /// - public void Dispose() - { + /// + public void Dispose() => // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method -#pragma warning disable IDE0022 // Use expression body for method Dispose(disposing: true); -#pragma warning restore IDE0022 // Use expression body for method + + private static FSDirectory? UnwrapSourceDirectory(Directory dir) + { + if (dir is SyncedFileSystemDirectory syncedDir) + { + return UnwrapDirectory(syncedDir.LocalLuceneDirectory); + } + + return UnwrapDirectory(dir); + } + + private static FSDirectory? UnwrapDirectory(Directory dir) + { + if (dir is FSDirectory fsDir) + { + return fsDir; + } + + if (dir is NRTCachingDirectory nrtDir) + { + return UnwrapSourceDirectory(nrtDir.Delegate); + } + + return null; } } } diff --git a/src/Examine.Lucene/ExamineTaxonomyReplicator.cs b/src/Examine.Lucene/ExamineTaxonomyReplicator.cs index 535e0e3c1..ec93a2ee6 100644 --- a/src/Examine.Lucene/ExamineTaxonomyReplicator.cs +++ b/src/Examine.Lucene/ExamineTaxonomyReplicator.cs @@ -10,6 +10,8 @@ namespace Examine.Lucene { + // TODO: Review all of this so that it is aligned with the fixes for the replicator changes. + /// /// Used to replicate an index to a destination directory /// @@ -60,7 +62,6 @@ public ExamineTaxonomyReplicator( var sourceDir = sourceIndex.GetLuceneDirectory() as FSDirectory; var destDir = destinationDirectory as FSDirectory; - var sourceTaxonomyDir = sourceIndex.GetLuceneTaxonomyDirectory() as FSDirectory; var destTaxonomyDir = destinationTaxonomyDirectory as FSDirectory; @@ -95,7 +96,7 @@ public void ReplicateIndex() IndexAndTaxonomyRevision rev; try { - rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.TaxonomyWriter as SnapshotDirectoryTaxonomyWriter); + rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.SnapshotDirectoryTaxonomyIndexWriterFactory); } catch (InvalidOperationException) { @@ -159,7 +160,7 @@ private void SourceIndex_IndexCommitted(object? sender, EventArgs? e) _logger.LogDebug("{IndexName} committed", index.Name); } } - var rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.TaxonomyWriter as SnapshotDirectoryTaxonomyWriter); + var rev = new IndexAndTaxonomyRevision(_sourceIndex.IndexWriter.IndexWriter, _sourceIndex.SnapshotDirectoryTaxonomyIndexWriterFactory); _replicator.Publish(rev); } diff --git a/src/Examine.Lucene/Indexing/DateTimeType.cs b/src/Examine.Lucene/Indexing/DateTimeType.cs index a0726e6e6..b7396d4a0 100644 --- a/src/Examine.Lucene/Indexing/DateTimeType.cs +++ b/src/Examine.Lucene/Indexing/DateTimeType.cs @@ -35,7 +35,7 @@ public class DateTimeType : IndexFieldRangeValueType, IIndexFacetValue public bool IsTaxonomyFaceted => _taxonomyIndex; /// - public DateTimeType(string fieldName, bool store, bool isFacetable, bool taxonomyIndex, ILoggerFactory logger, DateResolution resolution) + public DateTimeType(string fieldName, bool isFacetable, bool taxonomyIndex, ILoggerFactory logger, DateResolution resolution, bool store) : base(fieldName, logger, store) { Resolution = resolution; @@ -44,14 +44,12 @@ public DateTimeType(string fieldName, bool store, bool isFacetable, bool taxonom } /// - [Obsolete("To be removed in Examine V5")] + // [Obsolete("To be removed in Examine V5")] // TODO: Why? #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads public DateTimeType(string fieldName, ILoggerFactory logger, DateResolution resolution, bool store = true) #pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - : base(fieldName, logger, store) + : this(fieldName, false, false, logger, resolution, store) { - Resolution = resolution; - _isFacetable = false; } /// @@ -73,11 +71,12 @@ public override void AddValue(Document doc, object? value) var val = DateToLong(parsedVal); doc.Add(new Int64Field(FieldName, val, Store ? Field.Store.YES : Field.Store.NO)); - doc.Add(new FacetField(FieldName, parsedPathVal)); doc.Add(new NumericDocValuesField(FieldName, val)); + return; } + base.AddValue(doc, value); } diff --git a/src/Examine.Lucene/Indexing/DoubleType.cs b/src/Examine.Lucene/Indexing/DoubleType.cs index f050eab79..248a63a95 100644 --- a/src/Examine.Lucene/Indexing/DoubleType.cs +++ b/src/Examine.Lucene/Indexing/DoubleType.cs @@ -30,13 +30,12 @@ public DoubleType(string fieldName, bool isFacetable, bool taxonomyIndex, ILogge } /// - [Obsolete("To be removed in Examine V5")] + // [Obsolete("To be removed in Examine V5")] // TODO: Why? #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads public DoubleType(string fieldName, ILoggerFactory logger, bool store = true) #pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - : base(fieldName, logger, store) + : this(fieldName, false, false, logger, store) { - _isFacetable = false; } /// diff --git a/src/Examine.Lucene/Indexing/FullTextType.cs b/src/Examine.Lucene/Indexing/FullTextType.cs index afa50ef73..2ae33da06 100644 --- a/src/Examine.Lucene/Indexing/FullTextType.cs +++ b/src/Examine.Lucene/Indexing/FullTextType.cs @@ -45,7 +45,7 @@ public class FullTextType : IndexFieldValueTypeBase, IIndexFacetValueType /// /// Defaults to /// - public FullTextType(string fieldName, ILoggerFactory logger, bool isFacetable, bool taxonomyIndex, bool sortable, Analyzer analyzer) + public FullTextType(string fieldName, bool isFacetable, bool taxonomyIndex, bool sortable, ILoggerFactory logger, Analyzer analyzer) : base(fieldName, logger, true) { _sortable = sortable; @@ -63,15 +63,12 @@ public FullTextType(string fieldName, ILoggerFactory logger, bool isFacetable, b /// Defaults to /// /// - [Obsolete("To be removed in Examine V5")] + // [Obsolete("To be removed in Examine V5")] // TODO: Why? #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads public FullTextType(string fieldName, ILoggerFactory logger, Analyzer? analyzer = null, bool sortable = false) #pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - : base(fieldName, logger, true) + : this(fieldName, false, false, sortable, logger, analyzer ?? new CultureInvariantStandardAnalyzer()) { - _sortable = sortable; - _analyzer = analyzer ?? new CultureInvariantStandardAnalyzer(); - _isFacetable = false; } /// @@ -128,6 +125,7 @@ protected override void AddSingleValue(Document doc, object value) if (_sortable) { //to be sortable it cannot be analyzed so we have to make a different field + // TODO: Investigate https://lucene.apache.org/core/4_3_0/core/org/apache/lucene/document/SortedDocValuesField.html doc.Add(new StringField( ExamineFieldNames.SortedFieldNamePrefix + FieldName, str, diff --git a/src/Examine.Lucene/Indexing/GenericAnalyzerFieldValueType.cs b/src/Examine.Lucene/Indexing/GenericAnalyzerFieldValueType.cs index b6760e8f5..6620ed74f 100644 --- a/src/Examine.Lucene/Indexing/GenericAnalyzerFieldValueType.cs +++ b/src/Examine.Lucene/Indexing/GenericAnalyzerFieldValueType.cs @@ -41,6 +41,7 @@ protected override void AddSingleValue(Document doc, object value) if (_sortable) { //to be sortable it cannot be analyzed so we have to make a different field + // TODO: Investigate https://lucene.apache.org/core/4_3_0/core/org/apache/lucene/document/SortedDocValuesField.html doc.Add(new StringField( ExamineFieldNames.SortedFieldNamePrefix + FieldName, str, diff --git a/src/Examine.Lucene/Indexing/IIndexFieldValueType.cs b/src/Examine.Lucene/Indexing/IIndexFieldValueType.cs index de09fa5a8..10856b4f1 100644 --- a/src/Examine.Lucene/Indexing/IIndexFieldValueType.cs +++ b/src/Examine.Lucene/Indexing/IIndexFieldValueType.cs @@ -13,37 +13,37 @@ public interface IIndexFieldValueType /// /// The field name /// - string FieldName { get; } + public string FieldName { get; } /// /// Returns the sortable field name or null if the value isn't sortable /// /// By default it will not be sortable - string? SortableFieldName { get; } + public string? SortableFieldName { get; } /// /// Should the value be stored /// - bool Store { get; } + public bool Store { get; } /// /// Returns the analyzer for this field type, or null to use the default /// - Analyzer? Analyzer { get; } + public Analyzer? Analyzer { get; } /// /// Adds a value to the document /// /// /// - void AddValue(Document doc, object? value); + public void AddValue(Document doc, object? value); /// /// Gets a query as /// /// /// - Query? GetQuery(string query); + public Query? GetQuery(string query); //IHighlighter GetHighlighter(Query query, Searcher searcher, FacetsLoader facetsLoader); diff --git a/src/Examine.Lucene/Indexing/IndexFieldValueTypeBase.cs b/src/Examine.Lucene/Indexing/IndexFieldValueTypeBase.cs index 9709eca46..05f2e5a1b 100644 --- a/src/Examine.Lucene/Indexing/IndexFieldValueTypeBase.cs +++ b/src/Examine.Lucene/Indexing/IndexFieldValueTypeBase.cs @@ -74,9 +74,7 @@ private void AddSingleValueInternal(Document doc, object? value) /// /// protected bool TryConvert(object val, -#if !NETSTANDARD2_0 [MaybeNullWhen(false)] -#endif out T parsedVal) { // TODO: This throws all the time and then logs! diff --git a/src/Examine.Lucene/Indexing/Int32Type.cs b/src/Examine.Lucene/Indexing/Int32Type.cs index c82cd0231..a60cf1c74 100644 --- a/src/Examine.Lucene/Indexing/Int32Type.cs +++ b/src/Examine.Lucene/Indexing/Int32Type.cs @@ -30,13 +30,12 @@ public Int32Type(string fieldName,bool isFacetable, bool taxonomyIndex, ILoggerF } /// - [Obsolete("To be removed in Examine V5")] + // [Obsolete("To be removed in Examine V5")] // TODO: Why? #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads public Int32Type(string fieldName, ILoggerFactory logger, bool store = true) #pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - : base(fieldName, logger, store) + : this(fieldName, false, false, logger, store) { - _isFacetable = false; } /// @@ -69,6 +68,7 @@ public override void AddValue(Document doc, object? value) doc.Add(new NumericDocValuesField(FieldName, parsedVal)); return; } + base.AddValue(doc, value); } @@ -80,8 +80,16 @@ protected override void AddSingleValue(Document doc, object value) return; } - doc.Add(new Int32Field(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + // TODO: We can use this for better scoring/sorting performance + // https://stackoverflow.com/a/44953624/694494 + // https://lucene.apache.org/core/7_4_0/core/org/apache/lucene/document/NumericDocValuesField.html + //var dvField = new NumericDocValuesField(_docValuesFieldName, 0); + //dvField.SetInt32Value(parsedVal); + //doc.Add(dvField); + + doc.Add(new Int32Field(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + if (_isFacetable && _taxonomyIndex) { doc.Add(new FacetField(FieldName, parsedVal.ToString())); diff --git a/src/Examine.Lucene/Indexing/Int64Type.cs b/src/Examine.Lucene/Indexing/Int64Type.cs index 1218ff315..0f0c5ff6a 100644 --- a/src/Examine.Lucene/Indexing/Int64Type.cs +++ b/src/Examine.Lucene/Indexing/Int64Type.cs @@ -30,13 +30,12 @@ public Int64Type(string fieldName, bool isFacetable, bool taxonomyIndex, ILogger } /// - [Obsolete("To be removed in Examine V5")] + // [Obsolete("To be removed in Examine V5")] // TODO: Why? #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads public Int64Type(string fieldName, ILoggerFactory logger, bool store = true) #pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - : base(fieldName, logger, store) + : this(fieldName, false, false, logger, store) { - _isFacetable = false; } /// diff --git a/src/Examine.Lucene/Indexing/RawStringType.cs b/src/Examine.Lucene/Indexing/RawStringType.cs index b91fd30e2..d966ff2db 100644 --- a/src/Examine.Lucene/Indexing/RawStringType.cs +++ b/src/Examine.Lucene/Indexing/RawStringType.cs @@ -35,6 +35,12 @@ protected override void AddSingleValue(Document doc, object value) switch (value) { case IIndexableField f: + // https://lucene.apache.org/core/4_3_0/core/org/apache/lucene/index/IndexableField.html + // BinaryDocValuesField, ByteDocValuesField, DerefBytesDocValuesField, DoubleDocValuesField, DoubleField, + // Field, FloatDocValuesField, FloatField, IntDocValuesField, IntField, LongDocValuesField, LongField, + // NumericDocValuesField, PackedLongDocValuesField, ShortDocValuesField, SortedBytesDocValuesField, + // SortedDocValuesField, SortedSetDocValuesField, StoredField, StraightBytesDocValuesField, StringField, TextField + // https://solr.apache.org/guide/6_6/docvalues.html doc.Add(f); break; case TokenStream ts: diff --git a/src/Examine.Lucene/Indexing/SingleType.cs b/src/Examine.Lucene/Indexing/SingleType.cs index 859219d4b..d5aa3913f 100644 --- a/src/Examine.Lucene/Indexing/SingleType.cs +++ b/src/Examine.Lucene/Indexing/SingleType.cs @@ -31,13 +31,12 @@ public SingleType(string fieldName, bool isFacetable, bool taxonomyIndex, ILogge } /// - [Obsolete("To be removed in Examine V5")] + // [Obsolete("To be removed in Examine V5")] // TODO: Why? #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads public SingleType(string fieldName, ILoggerFactory logger, bool store = true) #pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - : base(fieldName, logger, store) + : this(fieldName, false, false, logger, store) { - _isFacetable = false; } /// diff --git a/src/Examine.Lucene/LoggingInfoStream.cs b/src/Examine.Lucene/LoggingInfoStream.cs index 314c3eca5..468c000a3 100644 --- a/src/Examine.Lucene/LoggingInfoStream.cs +++ b/src/Examine.Lucene/LoggingInfoStream.cs @@ -5,17 +5,21 @@ namespace Examine.Lucene { internal class LoggingInfoStream : InfoStream { - public LoggingInfoStream(ILogger logger) + private readonly LogLevel _logLevel; + + public LoggingInfoStream(ILogger logger, LogLevel logLevel) { Logger = logger; + _logLevel = logLevel; } public ILogger Logger { get; } - public override bool IsEnabled(string component) => Logger.IsEnabled(LogLevel.Debug); + public override bool IsEnabled(string component) => Logger.IsEnabled(_logLevel); + public override void Message(string component, string message) { - if (Logger.IsEnabled(LogLevel.Debug)) + if (Logger.IsEnabled(_logLevel)) { Logger.LogDebug("{Component} - {Message}", component, message); } diff --git a/src/Examine.Lucene/LoggingReplicationClient.cs b/src/Examine.Lucene/LoggingReplicationClient.cs index e1beafb02..6d8e6599e 100644 --- a/src/Examine.Lucene/LoggingReplicationClient.cs +++ b/src/Examine.Lucene/LoggingReplicationClient.cs @@ -29,7 +29,8 @@ protected override void HandleUpdateException(Exception exception) private class CustomLoggingInfoStream : LoggingInfoStream { - public CustomLoggingInfoStream(ILogger logger) : base(logger) + public CustomLoggingInfoStream(ILogger logger) + : base(logger, LogLevel.Debug) { } @@ -38,7 +39,7 @@ public override void Message(string component, string message) if (Logger.IsEnabled(LogLevel.Debug)) { // don't log this, it means there is no session - if (!message.EndsWith("=")) + if (!message.EndsWith('=')) { base.Message(component, message); } diff --git a/src/Examine.Lucene/LuceneIndexOptions.cs b/src/Examine.Lucene/LuceneIndexOptions.cs index 5c136c27d..0e564fe4c 100644 --- a/src/Examine.Lucene/LuceneIndexOptions.cs +++ b/src/Examine.Lucene/LuceneIndexOptions.cs @@ -1,6 +1,4 @@ -using System; using System.Collections.Generic; -using System.Text; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; using Lucene.Net.Facet; @@ -13,6 +11,34 @@ namespace Examine.Lucene /// public class LuceneIndexOptions : IndexOptions { + /// + /// Gets or sets whether Near Real-Time (NRT) indexing is enabled. + /// + public bool NrtEnabled { get; set; } = true; + + /// + /// Gets or sets the maximum stale seconds for Near Real-Time (NRT) indexing. + /// This defines the upper limit of staleness for NRT indexing operations. + /// + public double NrtTargetMaxStaleSec { get; set; } = 60.0; + + /// + /// Gets or sets the minimum stale seconds for Near Real-Time (NRT) indexing. + /// This defines the lower limit of staleness for NRT indexing operations. + /// + public double NrtTargetMinStaleSec { get; set; } = 1.0; + + /// + /// Gets or sets the maximum merge size in megabytes for the Near Real-Time (NRT) cache. + /// This defines the upper limit of memory usage for merging operations in the NRT cache. + /// + public double NrtCacheMaxMergeSizeMB { get; set; } = 5.0; + + /// + /// Gets or sets the maximum cached size in megabytes for the Near Real-Time (NRT) cache. + /// This defines the upper limit of memory usage for cached data in the NRT cache. + /// + public double NrtCacheMaxCachedMB { get; set; } = 60.0; /// /// THe index deletion policy /// @@ -37,10 +63,5 @@ public class LuceneIndexOptions : IndexOptions /// This is generally used to initialize any custom value types for your indexer since the value type collection cannot be modified at runtime. /// public IReadOnlyDictionary? IndexValueTypesFactory { get; set; } - - /// - /// Gets or Sets whether to use a Taxonomy Index - /// - public bool UseTaxonomyIndex { get; set; } } } diff --git a/src/Examine.Lucene/LuceneMultiSearcherOptions.cs b/src/Examine.Lucene/LuceneMultiSearcherOptions.cs new file mode 100644 index 000000000..884b1329d --- /dev/null +++ b/src/Examine.Lucene/LuceneMultiSearcherOptions.cs @@ -0,0 +1,15 @@ +using System; + +namespace Examine.Lucene +{ + /// + /// Examine Lucene MultiSearcher Configuration + /// + public class LuceneMultiSearcherOptions : LuceneSearcherOptions + { + /// + /// Index Names to search + /// + public string[] IndexNames { get; set; } = Array.Empty(); + } +} diff --git a/src/Examine.Lucene/LuceneSearcherOptions.cs b/src/Examine.Lucene/LuceneSearcherOptions.cs new file mode 100644 index 000000000..e0ac256dc --- /dev/null +++ b/src/Examine.Lucene/LuceneSearcherOptions.cs @@ -0,0 +1,22 @@ +using System; +using Lucene.Net.Analysis; +using Lucene.Net.Facet; + +namespace Examine.Lucene +{ + /// + /// Represents options for configuring a Lucene searcher. + /// + public class LuceneSearcherOptions + { + /// + /// Gets or sets the search analyzer. + /// + public Analyzer? Analyzer { get; set; } + + /// + /// Gets or sets the facet configuration. + /// + public FacetsConfig? FacetConfiguration { get; set; } + } +} diff --git a/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs b/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs index 223b49176..a3cfa2208 100644 --- a/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs +++ b/src/Examine.Lucene/Providers/BaseLuceneSearcher.cs @@ -1,9 +1,11 @@ using System; -using Lucene.Net.Analysis; -using Lucene.Net.Search; using Examine.Lucene.Search; using Examine.Search; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Standard; using Lucene.Net.Facet; +using Lucene.Net.Search; +using Microsoft.Extensions.Options; namespace Examine.Lucene.Providers { @@ -13,15 +15,11 @@ namespace Examine.Lucene.Providers public abstract class BaseLuceneSearcher : BaseSearchProvider, IDisposable { private readonly FacetsConfig _facetsConfig; - private bool _disposedValue; /// /// Constructor to allow for creating an indexer at runtime /// - /// - /// - [Obsolete("To remove in Examine V5")] - protected BaseLuceneSearcher(string name, Analyzer analyzer) + protected BaseLuceneSearcher(string name, IOptionsMonitor options) : base(name) { if (string.IsNullOrWhiteSpace(name)) @@ -29,26 +27,10 @@ protected BaseLuceneSearcher(string name, Analyzer analyzer) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); } - LuceneAnalyzer = analyzer; - _facetsConfig = GetDefaultFacetConfig(); - } - - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - /// - protected BaseLuceneSearcher(string name, Analyzer analyzer, FacetsConfig facetsConfig) - : base(name) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); - } + var searchOptions = options.Get(name); - LuceneAnalyzer = analyzer; - _facetsConfig = facetsConfig; + LuceneAnalyzer = searchOptions.Analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion); + _facetsConfig = searchOptions.FacetConfiguration ?? new FacetsConfig(); } /// @@ -57,9 +39,8 @@ protected BaseLuceneSearcher(string name, Analyzer analyzer, FacetsConfig facets public Analyzer LuceneAnalyzer { get; } /// - /// Gets the seach context + /// Gets the search context /// - /// public abstract ISearchContext GetSearchContext(); #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads @@ -93,33 +74,8 @@ public override ISearchResults Search(string searchText, QueryOptions? options = return sc.Execute(options); } - /// - /// Gets a FacetConfig with default configuration - /// - /// Facet Config - [Obsolete("To remove in Examine V5")] - public virtual FacetsConfig GetDefaultFacetConfig() => new FacetsConfig(); - - /// - /// Disposes of the searcher - /// - /// - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - _disposedValue = true; - } - } - /// - [Obsolete("The virutal modifier will be removed in Examine V5, override Dispose(bool) instead")] - public virtual void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } + public abstract void Dispose(); ///// ///// This is NOT used! however I'm leaving this here as example code diff --git a/src/Examine.Lucene/Providers/IIndexCommiter.cs b/src/Examine.Lucene/Providers/IIndexCommiter.cs deleted file mode 100644 index eae249958..000000000 --- a/src/Examine.Lucene/Providers/IIndexCommiter.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; - -namespace Examine.Lucene.Providers -{ - /// - /// This queues up a commit for the index so that a commit doesn't happen on every individual write since that is quite expensive - /// - public interface IIndexCommiter : IDisposable - { - /// - /// Commits the index to directory - /// - void CommitNow(); - - /// - /// Schedules the index to be commited to the directory - /// - void ScheduleCommit(); - } -} diff --git a/src/Examine.Lucene/Providers/IIndexCommitter.cs b/src/Examine.Lucene/Providers/IIndexCommitter.cs new file mode 100644 index 000000000..ebcb60866 --- /dev/null +++ b/src/Examine.Lucene/Providers/IIndexCommitter.cs @@ -0,0 +1,30 @@ +using System; + +namespace Examine.Lucene.Providers +{ + /// + /// This queues up a commit for the index so that a commit doesn't happen on every individual write since that is quite expensive + /// + public interface IIndexCommitter : IDisposable + { + /// + /// Commits the index to directory + /// + public void CommitNow(); + + /// + /// Schedules the index to be committed to the directory + /// + public void ScheduleCommit(); + + /// + /// Occurs when an error happens during the commit process. + /// + public event EventHandler CommitError; + + /// + /// Occurs when the index has been successfully committed. + /// + public event EventHandler? Committed; + } +} diff --git a/src/Examine.Lucene/Providers/ILuceneTaxonomySearcher.cs b/src/Examine.Lucene/Providers/ILuceneTaxonomySearcher.cs index 1b25fdee0..1df395d12 100644 --- a/src/Examine.Lucene/Providers/ILuceneTaxonomySearcher.cs +++ b/src/Examine.Lucene/Providers/ILuceneTaxonomySearcher.cs @@ -11,18 +11,18 @@ public interface ILuceneTaxonomySearcher : ISearcher, IDisposable /// /// The number of categories in the Taxonomy /// - int CategoryCount { get; } + public int CategoryCount { get; } /// /// Returns the Ordinal for the dim and path /// - int GetOrdinal(string dim, string[] path); + public int GetOrdinal(string dim, string[] path); /// /// Given a dimensions ordinal (id), get the Path. /// /// Demension ordinal (id) /// Path - IFacetLabel GetPath(int ordinal); + public IFacetLabel GetPath(int ordinal); } } diff --git a/src/Examine.Lucene/Providers/IndexCommitter.cs b/src/Examine.Lucene/Providers/IndexCommitter.cs new file mode 100644 index 000000000..413253897 --- /dev/null +++ b/src/Examine.Lucene/Providers/IndexCommitter.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Lucene.Net.Index; + +namespace Examine.Lucene.Providers +{ + /// + /// This queues up a commit for the index so that a commit doesn't happen on every individual write since that is quite expensive + /// + internal class IndexCommitter : DisposableObjectSlim, IIndexCommitter + { + private DateTime _timestamp; + private Timer? _timer; + private readonly object _locker = new object(); + private readonly LuceneIndex _index; + private readonly LuceneIndexOptions _indexOptions; + private readonly CancellationToken _cancellationToken; + private const int WaitMilliseconds = 1000; + + public IndexCommitter( + LuceneIndex index, + LuceneIndexOptions indexOptions, + CancellationToken cancellationToken) + { + _index = index; + _indexOptions = indexOptions; + _cancellationToken = cancellationToken; + } + + /// + /// The maximum time period that will elapse until we must commit (5 mins) + /// + private const int MaxWaitMilliseconds = 300000; + + public event EventHandler? CommitError; + public event EventHandler? Committed; + + /// + public void CommitNow() + { + _index.TaxonomyWriter.Commit(); + _index.IndexWriter.IndexWriter.Commit(); + Committed?.Invoke(this, EventArgs.Empty); + } + + /// + public void ScheduleCommit() + { + lock (_locker) + { + if (_timer == null) + { + //if we've been cancelled then be sure to commit now + if (_cancellationToken.IsCancellationRequested) + { + // perform the commit + CommitNow(); + } + else + { + //It's the initial call to this at the beginning or after successful commit + _timestamp = DateTime.Now; + _timer = new Timer(_ => TimerRelease()); + _timer.Change(WaitMilliseconds, 0); + } + } + else + { + //if we've been cancelled then be sure to cancel the timer and commit now + if (_cancellationToken.IsCancellationRequested) + { + //Stop the timer + _timer.Change(Timeout.Infinite, Timeout.Infinite); + _timer.Dispose(); + _timer = null; + + //perform the commit + CommitNow(); + } + else if ( + // must be less than the max + DateTime.Now - _timestamp < TimeSpan.FromMilliseconds(MaxWaitMilliseconds) && + // and less than the delay + DateTime.Now - _timestamp < TimeSpan.FromMilliseconds(WaitMilliseconds)) + { + //Delay + _timer.Change(WaitMilliseconds, 0); + } + else + { + //Cannot delay! the callback will execute on the pending timeout + } + } + } + } + + private void TimerRelease() + { + lock (_locker) + { + //if the timer is not null then a commit has been scheduled + if (_timer != null) + { + //Stop the timer + _timer.Change(Timeout.Infinite, Timeout.Infinite); + _timer.Dispose(); + _timer = null; + + try + { + //perform the commit + CommitNow(); + + // after the commit, refresh the searcher + _index.WaitForChanges(); + } + catch (Exception e) + { + // It is unclear how/why this happens but probably indicates index corruption + // see https://github.com/Shazwazza/Examine/issues/164 + CommitError?.Invoke(this, new IndexingErrorEventArgs( + _index, + "An error occurred during the index commit operation, if this error is persistent then index rebuilding is necessary", + "-1", + e)); + } + } + } + } + + protected override void DisposeResources() + { + TimerRelease(); + Committed = null; + CommitError = null; + } + } +} diff --git a/src/Examine.Lucene/Providers/LuceneIndex.cs b/src/Examine.Lucene/Providers/LuceneIndex.cs index 5a25b3c8a..0d5c83cee 100644 --- a/src/Examine.Lucene/Providers/LuceneIndex.cs +++ b/src/Examine.Lucene/Providers/LuceneIndex.cs @@ -2,25 +2,24 @@ using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; -using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Examine.Lucene.Directories; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Miscellaneous; +using Lucene.Net.Analysis.Standard; using Lucene.Net.Documents; +using Lucene.Net.Facet; +using Lucene.Net.Facet.Taxonomy; +using Lucene.Net.Facet.Taxonomy.Directory; using Lucene.Net.Index; +using Lucene.Net.Replicator; using Lucene.Net.Search; using Microsoft.Extensions.Logging; -using Directory = Lucene.Net.Store.Directory; -using static Lucene.Net.Index.IndexWriter; using Microsoft.Extensions.Options; -using Lucene.Net.Analysis.Standard; -using Examine.Lucene.Indexing; -using Examine.Lucene.Directories; -using Lucene.Net.Facet.Taxonomy; -using Lucene.Net.Facet.Taxonomy.Directory; -using static Lucene.Net.Replicator.IndexAndTaxonomyRevision; +using static Lucene.Net.Index.IndexWriter; +using Directory = Lucene.Net.Store.Directory; namespace Examine.Lucene.Providers { @@ -32,103 +31,6 @@ namespace Examine.Lucene.Providers /// public class LuceneIndex : BaseIndexProvider, IDisposable, IIndexStats, ReferenceManager.IRefreshListener { - #region Constructors - - /// - /// Constructor - /// - /// - /// - /// - /// - /// - /// - /// - protected LuceneIndex( - ILoggerFactory loggerFactory, - string name, - IOptionsMonitor indexOptions, - Func indexCommiterFactory, - IndexWriter? writer = null) - : base(loggerFactory, name, indexOptions) - { - _options = indexOptions.GetNamedOptions(name); - _committer = indexCommiterFactory(this); - _logger = loggerFactory.CreateLogger(); - - //initialize the field types - _fieldValueTypeCollection = new Lazy(() => CreateFieldValueTypes(_options.IndexValueTypesFactory)); - - if (_options.UseTaxonomyIndex) - { - _taxonomySearcher = new Lazy(CreateTaxonomySearcher); - - _searcher = new Lazy(() => _taxonomySearcher.Value); - } - else - { - _taxonomySearcher = new Lazy(() => throw new NotSupportedException("TaxonomySearcher not supported when not using taxonomy index.")); - _searcher = new Lazy(CreateSearcher); - } - - _cancellationTokenSource = new CancellationTokenSource(); - _cancellationToken = _cancellationTokenSource.Token; - - if (writer != null) - { - _writer = new TrackingIndexWriter(writer ?? throw new ArgumentNullException(nameof(writer))); - DefaultAnalyzer = writer.Analyzer; - } - else - { - DefaultAnalyzer = _options.Analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion); - } - var directoryOptions = indexOptions.GetNamedOptions(name); - - if (directoryOptions.DirectoryFactory == null) - { - throw new InvalidOperationException($"No {typeof(IDirectoryFactory)} assigned"); - } - - if (_options.UseTaxonomyIndex) - { - _taxonomyDirectory = new Lazy(() => directoryOptions.DirectoryFactory.CreateTaxonomyDirectory(this, directoryOptions.UnlockIndex)); - } - - _directory = new Lazy(() => directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex)); - } - - - private LuceneIndex( - ILoggerFactory loggerFactory, - string name, - IOptionsMonitor indexOptions) - : base(loggerFactory, name, indexOptions) - { - _options = indexOptions.GetNamedOptions(name); - _committer = new IndexCommiter(this); - _logger = loggerFactory.CreateLogger(); - - //initialize the field types - _fieldValueTypeCollection = new Lazy(() => CreateFieldValueTypes(_options.IndexValueTypesFactory)); - - if (_options.UseTaxonomyIndex) - { - _taxonomySearcher = new Lazy(CreateTaxonomySearcher); - - _searcher = new Lazy(() => _taxonomySearcher.Value); - } - else - { - _taxonomySearcher = new Lazy(() => throw new NotSupportedException("TaxonomySearcher not supported when not using taxonomy index.")); - _searcher = new Lazy(CreateSearcher); - } - _cancellationTokenSource = new CancellationTokenSource(); - _cancellationToken = _cancellationTokenSource.Token; - - DefaultAnalyzer = _options.Analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion); - } - /// /// Constructor to create an indexer /// @@ -136,7 +38,7 @@ public LuceneIndex( ILoggerFactory loggerFactory, string name, IOptionsMonitor indexOptions) - : this(loggerFactory, name, (IOptionsMonitor)indexOptions) + : this(loggerFactory, name, indexOptions, CreateDefaultCommitter) { var directoryOptions = indexOptions.GetNamedOptions(name); @@ -145,11 +47,14 @@ public LuceneIndex( throw new InvalidOperationException($"No {typeof(IDirectoryFactory)} assigned"); } - if (_options.UseTaxonomyIndex) + _lazyTaxonomyDirectory = new Lazy(() + => directoryOptions.DirectoryFactory.CreateTaxonomyDirectory(this, directoryOptions.UnlockIndex)); + + _lazyDirectory = new Lazy(() => { - _taxonomyDirectory = new Lazy(() => directoryOptions.DirectoryFactory.CreateTaxonomyDirectory(this, directoryOptions.UnlockIndex)); - } - _directory = new Lazy(() => directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex)); + _isDirectoryExternallyManaged = directoryOptions.DirectoryFactory is GenericDirectoryFactory gdf && gdf.ExternallyManaged; + return directoryOptions.DirectoryFactory.CreateDirectory(this, directoryOptions.UnlockIndex); + }); } //TODO: The problem with this is that the writer would already need to be configured with a PerFieldAnalyzerWrapper @@ -161,27 +66,50 @@ internal LuceneIndex( ILoggerFactory loggerFactory, string name, IOptionsMonitor indexOptions, - IndexWriter writer) - : this(loggerFactory, name, indexOptions) + IndexWriter writer, + SnapshotDirectoryTaxonomyIndexWriterFactory taxonomyWriterFactory) + : this(loggerFactory, name, indexOptions, CreateDefaultCommitter) { _writer = new TrackingIndexWriter(writer ?? throw new ArgumentNullException(nameof(writer))); + SnapshotDirectoryTaxonomyIndexWriterFactory = taxonomyWriterFactory ?? throw new ArgumentNullException(nameof(taxonomyWriterFactory)); + _lazyTaxonomyDirectory = new Lazy(() => SnapshotDirectoryTaxonomyIndexWriterFactory.IndexWriter.Directory); DefaultAnalyzer = writer.Analyzer; + _isDirectoryExternallyManaged = true; } - #endregion + private LuceneIndex( + ILoggerFactory loggerFactory, + string name, + IOptionsMonitor indexOptions, + Func indexCommiterFactory) + : base(loggerFactory, name, indexOptions) + { + _options = indexOptions.GetNamedOptions(name); + _logger = loggerFactory.CreateLogger(); + _committer = indexCommiterFactory(this); + + //initialize the field types + _fieldValueTypeCollection = new Lazy(() => CreateFieldValueTypes(_options.IndexValueTypesFactory)); + _taxonomySearcher = new Lazy(CreateSearcher); + _searcher = new Lazy(() => _taxonomySearcher.Value); + _cancellationTokenSource = new CancellationTokenSource(); + _cancellationToken = _cancellationTokenSource.Token; + DefaultAnalyzer = _options.Analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion); + } + + private static readonly string[] PossibleSuffixes = new[] { "Index", "Indexer" }; private readonly LuceneIndexOptions _options; private PerFieldAnalyzerWrapper? _fieldAnalyzer; - private ControlledRealTimeReopenThread? _nrtReopenThread; + private ControlledRealTimeReopenThread? _nrtReopenThread; private readonly ILogger _logger; - private readonly Lazy? _directory; -#if FULLDEBUG - private FileStream? _logOutput; -#endif + private readonly Lazy? _lazyDirectory; + private bool _isDirectoryExternallyManaged = false; private bool _disposedValue; - private readonly IIndexCommiter _committer; + private readonly IIndexCommitter _committer; private volatile TrackingIndexWriter? _writer; + private volatile DirectoryTaxonomyWriter? _taxonomyWriter; private int _activeWrites = 0; @@ -191,7 +119,7 @@ internal LuceneIndex( private readonly object _taskLocker = new object(); /// - /// Used to aquire the index writer + /// Used to acquire the index writer /// private readonly object _writerLocker = new object(); @@ -200,7 +128,7 @@ internal LuceneIndex( private bool? _exists; /// - /// Whether the Taxonomny Index exists. + /// Whether the Taxonomy Index exists. /// private bool? _taxonomyExists; @@ -234,11 +162,8 @@ internal LuceneIndex( // tracks the latest Generation value of what has been indexed.This can be used to force update a searcher to this generation. private long? _latestGen; - private volatile DirectoryTaxonomyWriter? _taxonomyWriter; - private ControlledRealTimeReopenThread? _taxonomyNrtReopenThread; - - private readonly Lazy _taxonomySearcher; - private readonly Lazy? _taxonomyDirectory; + private readonly Lazy? _taxonomySearcher; + private readonly Lazy? _lazyTaxonomyDirectory; #region Properties @@ -255,20 +180,10 @@ internal LuceneIndex( /// /// Gets the field ananlyzer /// - public PerFieldAnalyzerWrapper FieldAnalyzer - { - get - { - if (DefaultAnalyzer is PerFieldAnalyzerWrapper pfa) - { - return _fieldAnalyzer ??= pfa; - } - else - { - return _fieldAnalyzer ??= _fieldValueTypeCollection.Value.Analyzer; - } - } - } + public PerFieldAnalyzerWrapper FieldAnalyzer => (PerFieldAnalyzerWrapper)(_fieldAnalyzer ??= + (DefaultAnalyzer is PerFieldAnalyzerWrapper pfa) + ? pfa + : _fieldValueTypeCollection.Value.Analyzer); /// @@ -279,19 +194,16 @@ public PerFieldAnalyzerWrapper FieldAnalyzer public int CommitCount { get; protected internal set; } /// - /// Indicates whether or this system will process the queue items asynchonously - used for testing + /// Indicates whether or this system will process the queue items asynchronously - used for testing /// public bool RunAsync { get; protected internal set; } = true; - /// - /// This should ONLY be used internally by the scheduled committer we should refactor this out in the future - /// [EditorBrowsable(EditorBrowsableState.Never)] - protected bool IsCancellationRequested => _cancellationToken.IsCancellationRequested; + internal bool IsCancellationRequested => _cancellationToken.IsCancellationRequested; -#endregion + #endregion -#region Events + #region Events /// /// Occurs when [document writing]. @@ -299,17 +211,10 @@ public PerFieldAnalyzerWrapper FieldAnalyzer public event EventHandler? DocumentWriting; /// - /// Occors when the index is commited + /// Occurs when the index is committed /// public event EventHandler? IndexCommitted; - /// - /// Invokes the index commited event - /// - /// - /// - protected void RaiseIndexCommited(object sender, EventArgs e) => IndexCommitted?.Invoke(sender, e); - #endregion #region Event handlers @@ -337,9 +242,9 @@ protected override void OnIndexingError(IndexingErrorEventArgs e) protected virtual void OnDocumentWriting(DocumentWritingEventArgs docArgs) => DocumentWriting?.Invoke(this, docArgs); -#endregion + #endregion -#region Provider implementation + #region Provider implementation /// protected override void PerformIndexItems(IEnumerable values, Action onComplete) @@ -432,7 +337,7 @@ private int PerformIndexItemsInternal(IEnumerable valueSets, Cancellat /// public void EnsureIndex(bool forceOverwrite) { - if (!forceOverwrite && _exists.HasValue && _exists.Value && (!_options.UseTaxonomyIndex || (_taxonomyExists.HasValue && _taxonomyExists.Value))) + if (!forceOverwrite && _exists.HasValue && _exists.Value && _taxonomyExists.HasValue && _taxonomyExists.Value) { return; } @@ -457,13 +362,9 @@ public void EnsureIndex(bool forceOverwrite) //if there's no index, we need to create one CreateNewIndex(dir); - if (_options.UseTaxonomyIndex) - { - //Now create the taxonomy index - var taxonomyDir = GetLuceneTaxonomyDirectory() ?? throw new InvalidOperationException($"{Name} is configured to use a taxonomy index but the directory is null"); - - CreateNewTaxonomyIndex(taxonomyDir); - } + //Now create the taxonomy index + var taxonomyDir = GetLuceneTaxonomyDirectory(); + CreateNewTaxonomyIndex(taxonomyDir); } else { @@ -478,12 +379,8 @@ public void EnsureIndex(bool forceOverwrite) //In this case we need to initialize a writer and continue as normal. //Since we are already inside the writer lock and it is null, we are allowed to // make this call with out using GetIndexWriter() to do the initialization. - _writer ??= CreateIndexWriterInternal(); - - if (_options.UseTaxonomyIndex && _taxonomyWriter == null) - { - _taxonomyWriter = CreateTaxonomyWriterInternal(); - } + _writer ??= CreateIndexWriterWithLockCheck(); + _taxonomyWriter ??= CreateTaxonomyWriterWithLockCheck(); //We're forcing an overwrite, // this means that we need to cancel all operations currently in place, @@ -533,7 +430,7 @@ public void EnsureIndex(bool forceOverwrite) /// /// Used internally to create a brand new index, this is not thread safe /// - private void CreateNewIndex(Directory? dir) + private void CreateNewIndex(Directory dir) { IndexWriter? writer = null; try @@ -543,14 +440,14 @@ private void CreateNewIndex(Directory? dir) //unlock it! Unlock(dir); } + //create the writer (this will overwrite old index files) - var writerConfig = new IndexWriterConfig(LuceneInfo.CurrentVersion, FieldAnalyzer) - { - OpenMode = OpenMode.CREATE, - MergeScheduler = new ErrorLoggingConcurrentMergeScheduler(Name, - (s, e) => OnIndexingError(new IndexingErrorEventArgs(this, s, "-1", e))) - }; - writer = new IndexWriter(dir, writerConfig); + writer = CreateIndexWriterWithOpenMode(dir, OpenMode.CREATE); + + // Required to remove old index files which can be problematic + // if they remain in the index folder when replication is attempted. + writer.Commit(); + writer.WaitForMerges(); } catch (Exception ex) @@ -579,6 +476,7 @@ private void CreateNewTaxonomyIndex(Directory dir) Unlock(dir); } //create the writer (this will overwrite old index files) + // TODO: Surely we need to re-create the factory too since it hangs on to the writer :/ writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE); } catch (Exception ex) @@ -697,9 +595,9 @@ private int PerformDeleteFromIndexInternal(IEnumerable itemIds, Cancella return indexedNodes; } -#endregion + #endregion -#region Protected + #region Protected @@ -742,10 +640,6 @@ protected virtual FieldValueTypeCollection CreateFieldValueTypes(IReadOnlyDictio public override bool IndexExists() { var mainIndexExists = _writer != null || IndexExistsImpl(); - if (!_options.UseTaxonomyIndex) - { - return mainIndexExists; - } var taxonomyIndexExists = _taxonomyWriter != null || TaxonomyIndexExistsImpl(); return taxonomyIndexExists && mainIndexExists; } @@ -840,8 +734,6 @@ private bool TaxonomyIndexExistsImpl() return _taxonomyExists.Value; } - - /// /// Removes the specified term from the index /// @@ -908,7 +800,7 @@ protected virtual void AddDocument(Document doc, ValueSet valueSet) var indexTypeValueType = FieldValueTypeCollection.GetValueType(ExamineFieldNames.ItemTypeFieldName, FieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.InvariantCultureIgnoreCase)); indexTypeValueType.AddValue(doc, valueSet.ItemType); - if(valueSet.Values != null) + if (valueSet.Values != null) { foreach (var field in valueSet.Values) { @@ -960,138 +852,19 @@ protected virtual void AddDocument(Document doc, ValueSet valueSet) } // TODO: try/catch with OutOfMemoryException (see docs on UpdateDocument), though i've never seen this in real life - if (_options.UseTaxonomyIndex) - { - _latestGen = IndexWriter.UpdateDocument(new Term(ExamineFieldNames.ItemIdFieldName, valueSet.Id), _options.FacetsConfig.Build(TaxonomyWriter, doc)); - } - else - { - _latestGen = IndexWriter.UpdateDocument(new Term(ExamineFieldNames.ItemIdFieldName, valueSet.Id), _options.FacetsConfig.Build(doc)); - } + _latestGen = UpdateLuceneDocument(new Term(ExamineFieldNames.ItemIdFieldName, valueSet.Id), doc); } /// - /// This queues up a commit for the index so that a commit doesn't happen on every individual write since that is quite expensive + /// Updates a Lucene document in the index. /// - private class IndexCommiter : DisposableObjectSlim, IIndexCommiter - { - private readonly LuceneIndex _index; - private DateTime _timestamp; - private Timer? _timer; - private readonly object _locker = new object(); - private const int WaitMilliseconds = 2000; - - /// - /// The maximum time period that will elapse until we must commit (5 mins) - /// - private const int MaxWaitMilliseconds = 300000; - - /// - /// Constructor - /// - /// Index to commit - public IndexCommiter(LuceneIndex index) - { - _index = index; - } - - /// - public void CommitNow() - { - _index._taxonomyWriter?.Commit(); - _index._writer?.IndexWriter?.Commit(); - _index.IndexCommitted?.Invoke(_index, EventArgs.Empty); - } - - /// - public void ScheduleCommit() - { - lock (_locker) - { - if (_timer == null) - { - //if we've been cancelled then be sure to commit now - if (_index.IsCancellationRequested) - { - // perform the commit - CommitNow(); - } - else - { - //It's the initial call to this at the beginning or after successful commit - _timestamp = DateTime.Now; - _timer = new Timer(_ => TimerRelease()); - _timer.Change(WaitMilliseconds, 0); - } - } - else - { - //if we've been cancelled then be sure to cancel the timer and commit now - if (_index.IsCancellationRequested) - { - //Stop the timer - _timer.Change(Timeout.Infinite, Timeout.Infinite); - _timer.Dispose(); - _timer = null; - - //perform the commit - CommitNow(); - } - else if ( - // must be less than the max - DateTime.Now - _timestamp < TimeSpan.FromMilliseconds(MaxWaitMilliseconds) && - // and less than the delay - DateTime.Now - _timestamp < TimeSpan.FromMilliseconds(WaitMilliseconds)) - { - //Delay - _timer.Change(WaitMilliseconds, 0); - } - else - { - //Cannot delay! the callback will execute on the pending timeout - } - } - } - } - - - private void TimerRelease() - { - lock (_locker) - { - //if the timer is not null then a commit has been scheduled - if (_timer != null) - { - //Stop the timer - _timer.Change(Timeout.Infinite, Timeout.Infinite); - _timer.Dispose(); - _timer = null; - - try - { - //perform the commit - CommitNow(); - - // after the commit, refresh the searcher - _index.WaitForChanges(); - } - catch (Exception e) - { - // It is unclear how/why this happens but probably indicates index corruption - // see https://github.com/Shazwazza/Examine/issues/164 - _index.OnIndexingError(new IndexingErrorEventArgs( - _index, - "An error occurred during the index commit operation, if this error is persistent then index rebuilding is necessary", - "-1", - e)); - } - } - } - } - - protected override void DisposeResources() => TimerRelease(); - } - + /// The term used to identify the document to update. + /// The document to be updated in the index. + /// + /// The generation number of the update operation, or null if the operation is not applicable. + /// + protected virtual long? UpdateLuceneDocument(Term term, Document doc) + => IndexWriter.UpdateDocument(term, _options.FacetsConfig.Build(TaxonomyWriter, doc)); private bool ProcessQueueItem(IndexOperation item) { @@ -1113,33 +886,34 @@ private bool ProcessQueueItem(IndexOperation item) /// Returns the Lucene Directory used to store the index /// /// - public Directory? GetLuceneDirectory() => _writer != null ? _writer.IndexWriter.Directory : _directory?.Value; + public Directory GetLuceneDirectory() => _writer != null ? _writer.IndexWriter.Directory : _lazyDirectory?.Value ?? throw new InvalidOperationException($"{Name} does not have a Lucene Writer or Directory configured."); /// /// Returns the Lucene Directory used to store the taxonomy index /// /// - public Directory? GetLuceneTaxonomyDirectory() => _taxonomyWriter != null ? _taxonomyWriter.Directory : _taxonomyDirectory?.Value; + public Directory GetLuceneTaxonomyDirectory() => _taxonomyWriter != null ? _taxonomyWriter.Directory : _lazyTaxonomyDirectory?.Value ?? throw new InvalidOperationException($"{Name} does not have a Lucene Taxonomy Writer or Directory configured."); /// /// Used to create an index writer - this is called in GetIndexWriter (and therefore, GetIndexWriter should not be overridden) /// /// - private TrackingIndexWriter? CreateIndexWriterInternal() + private TrackingIndexWriter? CreateIndexWriterWithLockCheck() { var dir = GetLuceneDirectory(); - // Unfortunatley if the appdomain is taken down this will remain locked, so we can + // Unfortunately if the appdomain is taken down this will remain locked, so we can // ensure that it's unlocked here in that case. try { if (IsLocked(dir)) { - if (_logger.IsEnabled(LogLevel.Debug)) + if (_logger.IsEnabled(LogLevel.Information)) { _logger.LogDebug("Forcing index {IndexName} to be unlocked since it was left in a locked state", Name); } + //unlock it! Unlock(dir); } @@ -1162,46 +936,7 @@ private bool ProcessQueueItem(IndexOperation item) /// /// /// - protected virtual IndexWriter CreateIndexWriter(Directory? d) - { - if (d == null) - { - throw new ArgumentNullException(nameof(d)); - } - - var writer = new IndexWriter(d, new IndexWriterConfig(LuceneInfo.CurrentVersion, FieldAnalyzer) - { - IndexDeletionPolicy = _options.IndexDeletionPolicy ?? new KeepOnlyLastCommitDeletionPolicy(), -#if FULLDEBUG - - //If we want to enable logging of lucene output.... - //It is also possible to set a default InfoStream on the static IndexWriter class - InfoStream = - - _logOutput?.Close(); - if (LuceneIndexFolder != null) - { - try - { - System.IO.Directory.CreateDirectory(LuceneIndexFolder.FullName); - _logOutput = new FileStream(Path.Combine(LuceneIndexFolder.FullName, DateTime.UtcNow.ToString("yyyy-MM-dd") + ".log"), FileMode.Append); - - - } - catch (Exception ex) - { - //if an exception is thrown here we won't worry about it, it will mean we cannot create the log file - } - } - -#endif - - MergeScheduler = new ErrorLoggingConcurrentMergeScheduler(Name, - (s, e) => OnIndexingError(new IndexingErrorEventArgs(this, s, "-1", e))) - }); - - return writer; - } + protected virtual IndexWriter CreateIndexWriter(Directory d) => CreateIndexWriterWithOpenMode(d, OpenMode.CREATE_OR_APPEND); /// /// Gets the TrackingIndexWriter for the current directory @@ -1213,9 +948,6 @@ protected virtual IndexWriter CreateIndexWriter(Directory? d) /// See example: http://www.lucenetutorial.com/lucene-nrt-hello-world.html /// http://blog.mikemccandless.com/2011/11/near-real-time-readers-with-lucenes.html /// https://stackoverflow.com/questions/17993960/lucene-4-4-0-new-controlledrealtimereopenthread-sample-usage - /// TODO: Do we need/want to use the ControlledRealTimeReopenThread? Else according to mikecandles above in comments - /// we can probably just get away with using MaybeReopen each time we search. Though there are comments in the lucene - /// code to avoid that and do that on a background thread, which is exactly what ControlledRealTimeReopenThread already does. /// public TrackingIndexWriter IndexWriter { @@ -1228,7 +960,7 @@ public TrackingIndexWriter IndexWriter Monitor.Enter(_writerLocker); try { - _writer ??= CreateIndexWriterInternal(); + _writer ??= CreateIndexWriterWithLockCheck(); } finally { @@ -1245,11 +977,11 @@ public TrackingIndexWriter IndexWriter /// Used to create an index writer - this is called in GetIndexWriter (and therefore, GetIndexWriter should not be overridden) /// /// - private DirectoryTaxonomyWriter? CreateTaxonomyWriterInternal() + private DirectoryTaxonomyWriter? CreateTaxonomyWriterWithLockCheck() { var dir = GetLuceneTaxonomyDirectory(); - // Unfortunatley if the appdomain is taken down this will remain locked, so we can + // Unfortunately if the AppDomain is taken down this will remain locked, so we can // ensure that it's unlocked here in that case. try { @@ -1269,26 +1001,16 @@ public TrackingIndexWriter IndexWriter return null; } - var writer = CreateTaxonomyWriter(dir); - - return writer; + return new DirectoryTaxonomyWriter(SnapshotDirectoryTaxonomyIndexWriterFactory, dir); } /// - /// Method that creates the IndexWriter + /// Gets the taxonomy writer factory for the current index /// - /// - /// - protected virtual DirectoryTaxonomyWriter CreateTaxonomyWriter(Directory? d) - { - if (d == null) - { - throw new ArgumentNullException(nameof(d)); - } - var taxonomyWriter = new SnapshotDirectoryTaxonomyWriter(d); - - return taxonomyWriter; - } + /// + /// Due to strange lucene APIs, this factory actually hangs on to the index writer underneath and needs to be shared this way. + /// + internal SnapshotDirectoryTaxonomyIndexWriterFactory SnapshotDirectoryTaxonomyIndexWriterFactory { get; } = new SnapshotDirectoryTaxonomyIndexWriterFactory(); /// /// Gets the taxonomy writer for the current index @@ -1304,7 +1026,7 @@ public DirectoryTaxonomyWriter TaxonomyWriter Monitor.Enter(_writerLocker); try { - _taxonomyWriter ??= CreateTaxonomyWriterInternal(); + _taxonomyWriter ??= CreateTaxonomyWriterWithLockCheck(); } finally { @@ -1319,74 +1041,108 @@ public DirectoryTaxonomyWriter TaxonomyWriter #endregion -#region Private - - private LuceneSearcher CreateSearcher() + private IndexWriter CreateIndexWriterWithOpenMode(Directory d, OpenMode openMode) { - var possibleSuffixes = new[] { "Index", "Indexer" }; - var name = Name; - foreach (var suffix in possibleSuffixes) + if (d == null) { - //trim the "Indexer" / "Index" suffix if it exists - if (!name.EndsWith(suffix)) - { - continue; - } -#pragma warning disable IDE0057 // Use range operator - name = name.Substring(0, name.LastIndexOf(suffix, StringComparison.Ordinal)); -#pragma warning restore IDE0057 // Use range operator + throw new ArgumentNullException(nameof(d)); } - var writer = IndexWriter; - var searcherManager = new SearcherManager(writer.IndexWriter, true, new SearcherFactory()); - searcherManager.AddListener(this); - - _nrtReopenThread = new ControlledRealTimeReopenThread(writer, searcherManager, 5.0, 1.0) + var writerConfig = new IndexWriterConfig(LuceneInfo.CurrentVersion, FieldAnalyzer) { - Name = $"{Name} NRT Reopen Thread", - IsBackground = true + IndexDeletionPolicy = _options.IndexDeletionPolicy ?? new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), + OpenMode = openMode, + + //https://solr.apache.org/guide/solr/latest/configuration-guide/index-segments-merging.html#mergepolicyfactory + MergePolicy = new TieredMergePolicy + { + MaxMergeAtOnce = 10, + SegmentsPerTier = 10, + ForceMergeDeletesPctAllowed = 10.0f, + MaxMergedSegmentMB = 5000 + }, + MergeScheduler = new ErrorLoggingConcurrentMergeScheduler(Name, + (s, e) => OnIndexingError(new IndexingErrorEventArgs(this, s, "-1", e))) }; - _nrtReopenThread.Start(); + writerConfig.SetInfoStream(new LoggingInfoStream(_logger, LogLevel.Trace)); - // wait for most recent changes when first creating the searcher - WaitForChanges(); + if (_options.NrtEnabled) + { + // With NRT, we should apparently use this but there is no real implementation of it!? + // https://stackoverflow.com/questions/12271614/lucene-net-indexwriter-setmergedsegmentwarmer + writerConfig.MergedSegmentWarmer = new SimpleMergedSegmentWarmer(new LoggingInfoStream(_logger, LogLevel.Debug)); + } - return new LuceneSearcher(name + "Searcher", searcherManager, FieldAnalyzer, FieldValueTypeCollection, _options.FacetsConfig); + var writer = new IndexWriter(d, writerConfig); + + return writer; } - private LuceneTaxonomySearcher CreateTaxonomySearcher() + private LuceneSearcher CreateSearcher() { - var possibleSuffixes = new[] { "Index", "Indexer" }; var name = Name; - foreach (var suffix in possibleSuffixes) + foreach (var suffix in PossibleSuffixes) { //trim the "Indexer" / "Index" suffix if it exists if (!name.EndsWith(suffix)) - { + { continue; } -#pragma warning disable IDE0057 // Use range operator - name = name.Substring(0, name.LastIndexOf(suffix, StringComparison.Ordinal)); -#pragma warning restore IDE0057 // Use range operator + + name = name[..name.LastIndexOf(suffix, StringComparison.Ordinal)]; } var writer = IndexWriter; var taxonomyWriter = TaxonomyWriter; - var searcherManager = new SearcherTaxonomyManager(writer.IndexWriter, true, new SearcherFactory(), taxonomyWriter); + + // Create an IndexSearcher ReferenceManager to safely share IndexSearcher instances across + // multiple threads + var searcherManager = new SearcherTaxonomyManager( + writer.IndexWriter, + + // TODO: Apply All Deletes? Will be faster if this is false, https://blog.mikemccandless.com/2011/11/near-real-time-readers-with-lucenes.html + // BUT ... to do that we would need to fulfill this requirement: + // "yet during searching you have some way to ignore the old versions" + // Without fulfilling that requirement our Index_Read_And_Write_Ensure_No_Errors_In_Async tests fail when using + // non in-memory directories because it will return more results than what is actually in the index. + true, + new SearcherFactory(), + taxonomyWriter); + searcherManager.AddListener(this); - _taxonomyNrtReopenThread = new ControlledRealTimeReopenThread(writer, searcherManager, 5.0, 1.0) - { - Name = $"{Name} Taxonomy NRT Reopen Thread", - IsBackground = true - }; - _taxonomyNrtReopenThread.Start(); + if (_options.NrtEnabled) + { + // Create the ControlledRealTimeReopenThread that reopens the index periodically having into + // account the changes made to the index and tracked by the TrackingIndexWriter instance + // The index is refreshed every XX sec when nobody is waiting + // and every XX sec whenever is someone waiting (see search method) + // (see http://lucene.apache.org/core/4_3_0/core/org/apache/lucene/search/NRTManagerReopenThread.html) + _nrtReopenThread = new ControlledRealTimeReopenThread( + writer, + searcherManager, + _options.NrtTargetMaxStaleSec, // when there is nobody waiting + _options.NrtTargetMinStaleSec) // when there is someone waiting + { + Name = $"{Name} NRT Reopen Thread", + IsBackground = true + }; + + _nrtReopenThread.Start(); + // wait for most recent changes when first creating the searcher + WaitForChanges(); + } + else + { + // wait for most recent changes when first creating the searcher + searcherManager.MaybeRefreshBlocking(); + } // wait for most recent changes when first creating the searcher WaitForChanges(); - return new LuceneTaxonomySearcher(name + "Searcher", searcherManager, FieldAnalyzer, FieldValueTypeCollection, _options.FacetsConfig); + return new LuceneSearcher(name + "Searcher", searcherManager, FieldValueTypeCollection, new SearcherOptions(FieldAnalyzer, _options.FacetsConfig), _options.NrtEnabled); } /// @@ -1420,6 +1176,13 @@ private bool ProcessIndexQueueItem(IndexOperation op) return false; } + // TODO: We can re-use the same document object to save a lot of GC! + // https://cwiki.apache.org/confluence/display/lucene/ImproveIndexingSpeed + // Re-use Document and Field instances + // As of Lucene 2.3 there are new setValue(...) methods that allow you to change the value of a Field.This allows you to re - use a single Field instance across many added documents, which can save substantial GC cost. + // It's best to create a single Document instance, then add multiple Field instances to it, but hold onto these Field instances and re-use them by changing their values for each added document. For example you might have an idField, bodyField, nameField, storedField1, etc. After the document is added, you then directly change the Field values (idField.setValue(...), etc), and then re-add your Document instance. + // Note that you cannot re - use a single Field instance within a Document, and, you should not change a Field's value until the Document containing that Field has been added to the index. See Field for details. + var d = new Document(); AddDocument(d, indexingNodeDataArgs.ValueSet); @@ -1460,8 +1223,11 @@ private void QueueTask(Func op, Action onComplete, var indexedCount = 0; try { - // execute the callback - indexedCount = op(); + if (!currentToken.IsCancellationRequested) + { + // execute the callback + indexedCount = op(); + } } catch (Exception ex) { @@ -1504,8 +1270,6 @@ private void QueueTask(Func op, Action onComplete, } } -#endregion - /// /// Blocks the calling thread until the internal searcher can see latest documents /// @@ -1517,10 +1281,13 @@ public void WaitForChanges() { if (_latestGen.HasValue && !_disposedValue && !_cancellationToken.IsCancellationRequested) { - var found = _nrtReopenThread?.WaitForGeneration(_latestGen.Value, 5000); - if (_logger.IsEnabled(LogLevel.Debug)) + if (_options.NrtEnabled) { - _logger.LogDebug("{IndexName} WaitForChanges returned {GenerationFound}", Name, found); + var found = _nrtReopenThread?.WaitForGeneration(_latestGen.Value, 5000); + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("{IndexName} WaitForChanges returned {GenerationFound}", Name, found); + } } } } @@ -1587,34 +1354,36 @@ private bool RetryUntilSuccessOrTimeout(Func task, TimeSpan timeout, TimeS return false; } + private static IIndexCommitter CreateDefaultCommitter(LuceneIndex index) + { + var committer = new IndexCommitter(index, index._options, index._cancellationToken); + committer.CommitError += (sender, args) => index.OnIndexingError(args); + committer.Committed += (sender, args) => index.IndexCommitted?.Invoke(index, EventArgs.Empty); + return committer; + } + /// + // TODO: Why is this obsolete?? protected virtual void Dispose(bool disposing) { if (!_disposedValue) { if (disposing) { + //cancel any operation currently in place + _cancellationTokenSource.Cancel(); + if (_nrtReopenThread is not null) { _nrtReopenThread.Interrupt(); _nrtReopenThread.Dispose(); } - // The type of _taxonomyNrtReopenThread has overriden the != operator and expects a non null value to compare the references. Therefore we use is not null instead of != null. - if (_taxonomyNrtReopenThread is not null) - { - _taxonomyNrtReopenThread.Interrupt(); - _taxonomyNrtReopenThread.Dispose(); - } - if (_searcher != null && _searcher.IsValueCreated) { _searcher.Value.Dispose(); } - //cancel any operation currently in place - _cancellationTokenSource.Cancel(); - //Don't close the writer until there are definitely no more writes //NOTE: we are not taking into acccount the WaitForIndexQueueOnShutdown property here because we really want to make sure //we are not terminating Lucene while it is actively writing to the index. @@ -1627,6 +1396,7 @@ protected virtual void Dispose(bool disposing) { try { + // TODO: Is this the same analyzer we dispose below?? _writer?.IndexWriter?.Analyzer.Dispose(); } catch (ObjectDisposedException) @@ -1648,9 +1418,8 @@ protected virtual void Dispose(bool disposing) { OnIndexingError(new IndexingErrorEventArgs(this, "Error closing the index", "-1", e)); } - - } + if (_taxonomyWriter != null) { try @@ -1665,10 +1434,21 @@ protected virtual void Dispose(bool disposing) } _cancellationTokenSource.Dispose(); + _fieldAnalyzer?.Dispose(); + if (!object.ReferenceEquals(_fieldAnalyzer, DefaultAnalyzer)) + { + DefaultAnalyzer?.Dispose(); + } + + if ((_lazyDirectory?.IsValueCreated ?? false) && !_isDirectoryExternallyManaged) + { + _lazyDirectory.Value.Dispose(); + } -#if FULLDEBUG - _logOutput?.Close(); -#endif + if ((_lazyTaxonomyDirectory?.IsValueCreated ?? false) && !_isDirectoryExternallyManaged) + { + _lazyTaxonomyDirectory.Value.Dispose(); + } } _disposedValue = true; } @@ -1687,7 +1467,5 @@ void ReferenceManager.IRefreshListener.AfterRefresh(bool didRefresh) } } } - - } diff --git a/src/Examine.Lucene/Providers/LuceneSearcher.cs b/src/Examine.Lucene/Providers/LuceneSearcher.cs index c4010d5a4..45ae0e62c 100644 --- a/src/Examine.Lucene/Providers/LuceneSearcher.cs +++ b/src/Examine.Lucene/Providers/LuceneSearcher.cs @@ -1,75 +1,122 @@ -using System; using Examine.Lucene.Search; -using Lucene.Net.Search; -using Lucene.Net.Analysis; -using Lucene.Net.Facet; +using Examine.Search; +using Lucene.Net.Facet.Taxonomy; +using Lucene.Net.Index; +using Microsoft.Extensions.Options; namespace Examine.Lucene.Providers { - - /// - /// Standard object used to search a Lucene index - /// - public class LuceneSearcher : BaseLuceneSearcher, IDisposable + /// + /// A searcher for taxonomy indexes + /// + internal class LuceneSearcher : BaseLuceneSearcher, ILuceneTaxonomySearcher { - private readonly SearcherManager _searcherManager; + private readonly SearcherTaxonomyManager _searcherManager; private readonly FieldValueTypeCollection _fieldValueTypeCollection; + private readonly bool _isNrt; private bool _disposedValue; - + private volatile ITaxonomySearchContext? _searchContext; /// /// Constructor allowing for creating a NRT instance based on a given writer /// - /// - /// - /// - /// - [Obsolete("To remove in Examine V5")] - public LuceneSearcher(string name, SearcherManager searcherManager, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection) - : base(name, analyzer) + public LuceneSearcher(string name, SearcherTaxonomyManager searcherManager, FieldValueTypeCollection fieldValueTypeCollection, IOptionsMonitor options, bool isNrt) + : base(name, options) { _searcherManager = searcherManager; _fieldValueTypeCollection = fieldValueTypeCollection; + _isNrt = isNrt; + } + + /// + public override ISearchContext GetSearchContext() + { + // Don't create a new search context unless something has changed + var isCurrent = IsSearcherCurrent(_searcherManager); + if (_searchContext is null || !isCurrent) + { + _searchContext = new TaxonomySearchContext(_searcherManager, _fieldValueTypeCollection, _isNrt); + } + + return _searchContext; } /// - /// Constructor allowing for creating a NRT instance based on a given writer + /// Gets the Taxonomy SearchContext /// - /// - /// - /// - /// - /// - public LuceneSearcher(string name, SearcherManager searcherManager, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection, FacetsConfig facetsConfig) - : base(name, analyzer, facetsConfig) + /// + public virtual ITaxonomySearchContext GetTaxonomySearchContext() { - _searcherManager = searcherManager; - _fieldValueTypeCollection = fieldValueTypeCollection; - } + // Don't create a new search context unless something has changed + var isCurrent = IsSearcherCurrent(_searcherManager); + if (_searchContext is null || !isCurrent) + { + _searchContext = new TaxonomySearchContext(_searcherManager, _fieldValueTypeCollection, _isNrt); + } - /// - public override ISearchContext GetSearchContext() - => new SearchContext(_searcherManager, _fieldValueTypeCollection); + return _searchContext; + } - /// - [Obsolete("To remove in Examine v5")] - protected new virtual void Dispose(bool disposing) + /// + public override void Dispose() { if (!_disposedValue) { - if (disposing) - { - _searcherManager.Dispose(); - } - + _searcherManager.Dispose(); _disposedValue = true; } - base.Dispose(disposing); + ; } /// - [Obsolete("To remove in Examine V5 - IDisposable is implemented in base class")] - public new void Dispose() => Dispose(true); + public int CategoryCount + { + get + { + var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; + return taxonomyReader.Count; + } + } + + /// + public int GetOrdinal(string dimension, string[] path) + { + var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; + return taxonomyReader.GetOrdinal(dimension, path); + } + + + /// + public IFacetLabel GetPath(int ordinal) + { + var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; + var facetLabel = taxonomyReader.GetPath(ordinal); + var examineFacetLabel = new LuceneFacetLabel(facetLabel); + return examineFacetLabel; + } + + // + // Summary: + // Returns true if no changes have occured since this searcher ie. reader was opened, + // otherwise false. + private bool IsSearcherCurrent(SearcherTaxonomyManager searcherTaxonomyManager) + { + var indexSearcher = searcherTaxonomyManager.Acquire(); + try + { + var indexReader = indexSearcher.Searcher.IndexReader; + //if (Debugging.AssertsEnabled) + //{ + // Debugging.Assert(indexReader is DirectoryReader, "searcher's IndexReader should be a DirectoryReader, but got {0}", indexReader); + //} + + return ((DirectoryReader)indexReader).IsCurrent(); + } + finally + { + searcherTaxonomyManager.Release(indexSearcher); + } + } } } diff --git a/src/Examine.Lucene/Providers/LuceneTaxonomySearcher.cs b/src/Examine.Lucene/Providers/LuceneTaxonomySearcher.cs deleted file mode 100644 index 0ef607a09..000000000 --- a/src/Examine.Lucene/Providers/LuceneTaxonomySearcher.cs +++ /dev/null @@ -1,88 +0,0 @@ -using System; -using Examine.Lucene.Search; -using Examine.Search; -using Lucene.Net.Analysis; -using Lucene.Net.Facet; -using Lucene.Net.Facet.Taxonomy; - -namespace Examine.Lucene.Providers -{ - /// - /// A searcher for taxonomy indexes - /// - public class LuceneTaxonomySearcher : BaseLuceneSearcher, IDisposable, ILuceneTaxonomySearcher - { - private readonly SearcherTaxonomyManager _searcherManager; - private readonly FieldValueTypeCollection _fieldValueTypeCollection; - private bool _disposedValue; - - /// - /// Constructor allowing for creating a NRT instance based on a given writer - /// - /// - /// - /// - /// - /// - public LuceneTaxonomySearcher(string name, SearcherTaxonomyManager searcherManager, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection, FacetsConfig facetsConfig) - : base(name, analyzer, facetsConfig) - { - _searcherManager = searcherManager; - _fieldValueTypeCollection = fieldValueTypeCollection; - } - - /// - public override ISearchContext GetSearchContext() - => new TaxonomySearchContext(_searcherManager, _fieldValueTypeCollection); - - /// - /// Gets the Taxonomy SearchContext - /// - /// - public virtual ITaxonomySearchContext GetTaxonomySearchContext() - => new TaxonomySearchContext(_searcherManager, _fieldValueTypeCollection); - - /// - protected override void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - _searcherManager.Dispose(); - } - - _disposedValue = true; - } - base.Dispose(disposing); - } - - /// - public int CategoryCount - { - get - { - var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; - return taxonomyReader.Count; - } - } - - /// - public int GetOrdinal(string dimension, string[] path) - { - var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; - return taxonomyReader.GetOrdinal(dimension, path); - } - - - /// - public IFacetLabel GetPath(int ordinal) - { - var taxonomyReader = GetTaxonomySearchContext().GetTaxonomyAndSearcher().TaxonomyReader; - var facetLabel = taxonomyReader.GetPath(ordinal); - var examineFacetLabel = new LuceneFacetLabel(facetLabel); - return examineFacetLabel; - } - } -} - diff --git a/src/Examine.Lucene/Providers/MultiIndexSearcher.cs b/src/Examine.Lucene/Providers/MultiIndexSearcher.cs index 5f17b177f..7133d5002 100644 --- a/src/Examine.Lucene/Providers/MultiIndexSearcher.cs +++ b/src/Examine.Lucene/Providers/MultiIndexSearcher.cs @@ -2,9 +2,7 @@ using System.Collections.Generic; using System.Linq; using Examine.Lucene.Search; -using Lucene.Net.Analysis; -using Lucene.Net.Analysis.Standard; -using Lucene.Net.Facet; +using Microsoft.Extensions.Options; namespace Examine.Lucene.Providers { @@ -18,12 +16,8 @@ public class MultiIndexSearcher : BaseLuceneSearcher /// /// Constructor to allow for creating a searcher at runtime /// - /// - /// - /// - [Obsolete("To remove in Examine V5")] - public MultiIndexSearcher(string name, IEnumerable indexes, Analyzer? analyzer = null) - : base(name, analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion)) + public MultiIndexSearcher(string name, IOptionsMonitor options, IEnumerable indexes) + : base(name, options) { _searchers = new Lazy>(() => indexes.Select(x => x.Searcher)); } @@ -31,43 +25,8 @@ public MultiIndexSearcher(string name, IEnumerable indexes, Analyzer? an /// /// Constructor to allow for creating a searcher at runtime /// - /// - /// - /// - [Obsolete("To remove in Examine V5")] - public MultiIndexSearcher(string name, Lazy> searchers, Analyzer? analyzer = null) - : base(name, analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion)) - { - _searchers = searchers; - } - - - /// - /// Constructor to allow for creating a searcher at runtime - /// - /// - /// - /// Get the current by injecting or use new FacetsConfig() for an empty configuration - /// -#pragma warning disable RS0026 // Do not add multiple public overloads with optional parameters - public MultiIndexSearcher(string name, IEnumerable indexes, FacetsConfig facetsConfig, Analyzer? analyzer = null) - : base(name, analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion), facetsConfig) -#pragma warning restore RS0026 // Do not add multiple public overloads with optional parameters - { - _searchers = new Lazy>(() => indexes.Select(x => x.Searcher)); - } - - /// - /// Constructor to allow for creating a searcher at runtime - /// - /// - /// - /// Get the current by injecting or use new FacetsConfig() for an empty configuration - /// -#pragma warning disable RS0026 // Do not add multiple public overloads with optional parameters - public MultiIndexSearcher(string name, Lazy> searchers, FacetsConfig facetsConfig, Analyzer? analyzer = null) - : base(name, analyzer ?? new StandardAnalyzer(LuceneInfo.CurrentVersion), facetsConfig) -#pragma warning restore RS0026 // Do not add multiple public overloads with optional parameters + public MultiIndexSearcher(string name, IOptionsMonitor options, Lazy> searchers) + : base(name, options) { _searchers = searchers; } @@ -75,13 +34,18 @@ public MultiIndexSearcher(string name, Lazy> searchers, F /// /// The underlying LuceneSearchers that will be searched across /// - public IEnumerable Searchers => _searchers.Value.OfType(); + public IEnumerable Searchers => _searchers.Value.OfType(); /// /// Are the searchers initialized /// public bool SearchersInitialized => _searchers.IsValueCreated; + /// + public override void Dispose() + { + } + /// public override ISearchContext GetSearchContext() => new MultiSearchContext(Searchers.Select(s => s.GetSearchContext()).ToArray()); diff --git a/src/Examine.Lucene/Providers/SearcherOptions.cs b/src/Examine.Lucene/Providers/SearcherOptions.cs new file mode 100644 index 000000000..06792f42b --- /dev/null +++ b/src/Examine.Lucene/Providers/SearcherOptions.cs @@ -0,0 +1,28 @@ +using System; +using Lucene.Net.Analysis; +using Lucene.Net.Facet; +using Microsoft.Extensions.Options; + +namespace Examine.Lucene.Providers +{ + internal sealed class SearcherOptions : IOptionsMonitor + { + private readonly LuceneSearcherOptions _options; + + public SearcherOptions(Analyzer analyzer, FacetsConfig facetsConfig) + { + _options = new LuceneSearcherOptions + { + Analyzer = analyzer, + FacetConfiguration = facetsConfig + }; + } + + public LuceneSearcherOptions Get(string? name) => _options; + + public LuceneSearcherOptions CurrentValue => throw new NotSupportedException(); + + public IDisposable OnChange(Action listener) => throw new NotSupportedException(); + } +} + diff --git a/src/Examine.Lucene/PublicAPI.Shipped.txt b/src/Examine.Lucene/PublicAPI.Shipped.txt index ba7a27798..29157058a 100644 --- a/src/Examine.Lucene/PublicAPI.Shipped.txt +++ b/src/Examine.Lucene/PublicAPI.Shipped.txt @@ -1,5 +1,4 @@ #nullable enable -abstract Examine.Lucene.Directories.DirectoryFactoryBase.CreateDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! abstract Examine.Lucene.Indexing.IndexFieldRangeValueType.GetQuery(T? lower, T? upper, bool lowerInclusive = true, bool upperInclusive = true) -> Lucene.Net.Search.Query! abstract Examine.Lucene.Indexing.IndexFieldValueTypeBase.AddSingleValue(Lucene.Net.Documents.Document! doc, object! value) -> void abstract Examine.Lucene.Providers.BaseLuceneSearcher.GetSearchContext() -> Examine.Lucene.Search.ISearchContext! @@ -41,14 +40,7 @@ Examine.Lucene.DelegateFieldValueTypeFactory.DelegateFieldValueTypeFactory(Syste Examine.Lucene.Directories.DefaultLockFactory Examine.Lucene.Directories.DefaultLockFactory.DefaultLockFactory() -> void Examine.Lucene.Directories.DefaultLockFactory.GetLockFactory(System.IO.DirectoryInfo! directory) -> Lucene.Net.Store.LockFactory! -Examine.Lucene.Directories.DirectoryFactoryBase -Examine.Lucene.Directories.DirectoryFactoryBase.DirectoryFactoryBase() -> void -Examine.Lucene.Directories.DirectoryFactoryBase.Dispose() -> void -Examine.Lucene.Directories.FileSystemDirectoryFactory -Examine.Lucene.Directories.FileSystemDirectoryFactory.FileSystemDirectoryFactory(System.IO.DirectoryInfo! baseDir, Examine.Lucene.Directories.ILockFactory! lockFactory) -> void -Examine.Lucene.Directories.FileSystemDirectoryFactory.LockFactory.get -> Examine.Lucene.Directories.ILockFactory! Examine.Lucene.Directories.GenericDirectoryFactory -Examine.Lucene.Directories.GenericDirectoryFactory.GenericDirectoryFactory(System.Func! factory) -> void Examine.Lucene.Directories.IApplicationIdentifier Examine.Lucene.Directories.IApplicationIdentifier.GetApplicationUniqueIdentifier() -> string! Examine.Lucene.Directories.IDirectoryFactory @@ -59,16 +51,13 @@ Examine.Lucene.Directories.MultiIndexLockFactory Examine.Lucene.Directories.MultiIndexLockFactory.MultiIndexLockFactory(Lucene.Net.Store.Directory! master, Lucene.Net.Store.Directory! child) -> void Examine.Lucene.Directories.MultiIndexLockFactory.MultiIndexLockFactory(Lucene.Net.Store.LockFactory! master, Lucene.Net.Store.LockFactory! child) -> void Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory -Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo! localDir, System.IO.DirectoryInfo! mainDir, Examine.Lucene.Directories.ILockFactory! lockFactory, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void Examine.Lucene.Directories.TempEnvFileSystemDirectoryFactory -Examine.Lucene.Directories.TempEnvFileSystemDirectoryFactory.TempEnvFileSystemDirectoryFactory(Examine.Lucene.Directories.IApplicationIdentifier! applicationIdentifier, Examine.Lucene.Directories.ILockFactory! lockFactory) -> void Examine.Lucene.DocumentWritingEventArgs Examine.Lucene.DocumentWritingEventArgs.Document.get -> Lucene.Net.Documents.Document! Examine.Lucene.DocumentWritingEventArgs.DocumentWritingEventArgs(Examine.ValueSet! valueSet, Lucene.Net.Documents.Document! d) -> void Examine.Lucene.DocumentWritingEventArgs.ValueSet.get -> Examine.ValueSet! Examine.Lucene.ExamineReplicator Examine.Lucene.ExamineReplicator.Dispose() -> void -Examine.Lucene.ExamineReplicator.ExamineReplicator(Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Examine.Lucene.Providers.LuceneIndex! sourceIndex, Lucene.Net.Store.Directory! destinationDirectory, System.IO.DirectoryInfo! tempStorage) -> void Examine.Lucene.ExamineReplicator.ReplicateIndex() -> void Examine.Lucene.ExamineReplicator.StartIndexReplicationOnSchedule(int milliseconds) -> void Examine.Lucene.FieldValueTypeCollection @@ -136,7 +125,6 @@ Examine.Lucene.LuceneIndexOptions.IndexValueTypesFactory.get -> System.Collectio Examine.Lucene.LuceneIndexOptions.IndexValueTypesFactory.set -> void Examine.Lucene.LuceneIndexOptions.LuceneIndexOptions() -> void Examine.Lucene.Providers.BaseLuceneSearcher -Examine.Lucene.Providers.BaseLuceneSearcher.BaseLuceneSearcher(string! name, Lucene.Net.Analysis.Analyzer! analyzer) -> void Examine.Lucene.Providers.BaseLuceneSearcher.CreateQuery(string? category, Examine.Search.BooleanOperation defaultOperation, Lucene.Net.Analysis.Analyzer! luceneAnalyzer, Examine.Lucene.Search.LuceneSearchOptions! searchOptions) -> Examine.Search.IQuery! Examine.Lucene.Providers.BaseLuceneSearcher.LuceneAnalyzer.get -> Lucene.Net.Analysis.Analyzer! Examine.Lucene.Providers.ErrorCheckingScoringBooleanQueryRewrite @@ -155,24 +143,16 @@ Examine.Lucene.Providers.LuceneIndex.FieldAnalyzer.get -> Lucene.Net.Analysis.Mi Examine.Lucene.Providers.LuceneIndex.FieldValueTypeCollection.get -> Examine.Lucene.FieldValueTypeCollection! Examine.Lucene.Providers.LuceneIndex.GetDocumentCount() -> long Examine.Lucene.Providers.LuceneIndex.GetFieldNames() -> System.Collections.Generic.IEnumerable! -Examine.Lucene.Providers.LuceneIndex.GetLuceneDirectory() -> Lucene.Net.Store.Directory? Examine.Lucene.Providers.LuceneIndex.IndexCommitted -> System.EventHandler? Examine.Lucene.Providers.LuceneIndex.IndexReady() -> bool Examine.Lucene.Providers.LuceneIndex.IndexWriter.get -> Lucene.Net.Index.TrackingIndexWriter! -Examine.Lucene.Providers.LuceneIndex.IsCancellationRequested.get -> bool Examine.Lucene.Providers.LuceneIndex.IsReadable(out System.Exception? ex) -> bool Examine.Lucene.Providers.LuceneIndex.LuceneIndex(Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, string! name, Microsoft.Extensions.Options.IOptionsMonitor! indexOptions) -> void Examine.Lucene.Providers.LuceneIndex.RunAsync.get -> bool Examine.Lucene.Providers.LuceneIndex.RunAsync.set -> void Examine.Lucene.Providers.LuceneIndex.WaitForChanges() -> void Examine.Lucene.Providers.LuceneIndex.WithThreadingMode(Examine.Lucene.Providers.IndexThreadingMode mode) -> System.IDisposable! -Examine.Lucene.Providers.LuceneSearcher -Examine.Lucene.Providers.LuceneSearcher.Dispose() -> void -Examine.Lucene.Providers.LuceneSearcher.LuceneSearcher(string! name, Lucene.Net.Search.SearcherManager! searcherManager, Lucene.Net.Analysis.Analyzer! analyzer, Examine.Lucene.FieldValueTypeCollection! fieldValueTypeCollection) -> void Examine.Lucene.Providers.MultiIndexSearcher -Examine.Lucene.Providers.MultiIndexSearcher.MultiIndexSearcher(string! name, System.Collections.Generic.IEnumerable! indexes, Lucene.Net.Analysis.Analyzer? analyzer = null) -> void -Examine.Lucene.Providers.MultiIndexSearcher.MultiIndexSearcher(string! name, System.Lazy!>! searchers, Lucene.Net.Analysis.Analyzer? analyzer = null) -> void -Examine.Lucene.Providers.MultiIndexSearcher.Searchers.get -> System.Collections.Generic.IEnumerable! Examine.Lucene.Providers.MultiIndexSearcher.SearchersInitialized.get -> bool Examine.Lucene.Providers.ValueSetValidatorDelegate Examine.Lucene.Providers.ValueSetValidatorDelegate.Validate(Examine.ValueSet! valueSet) -> Examine.ValueSetValidationResult @@ -226,7 +206,6 @@ Examine.Lucene.Search.LuceneQuery.ManagedQuery(string! query, string![]? fields Examine.Lucene.Search.LuceneQuery.NativeQuery(string! query) -> Examine.Search.IBooleanOperation! Examine.Lucene.Search.LuceneQuery.RangeQuery(string![]! fields, T? min, T? max, bool minInclusive = true, bool maxInclusive = true) -> Examine.Search.IBooleanOperation! Examine.Lucene.Search.LuceneQueryOptions -Examine.Lucene.Search.LuceneQueryOptions.LuceneQueryOptions(int skip, int? take = null, Examine.Lucene.Search.SearchAfterOptions? searchAfter = null, bool trackDocumentScores = false, bool trackDocumentMaxScore = false) -> void Examine.Lucene.Search.LuceneQueryOptions.SearchAfter.get -> Examine.Lucene.Search.SearchAfterOptions? Examine.Lucene.Search.LuceneQueryOptions.TrackDocumentMaxScore.get -> bool Examine.Lucene.Search.LuceneQueryOptions.TrackDocumentScores.get -> bool @@ -294,7 +273,6 @@ Examine.Lucene.Search.LuceneSearchResult.LuceneSearchResult(string! id, float sc Examine.Lucene.Search.LuceneSearchResult.ShardIndex.get -> int Examine.Lucene.Search.LuceneSearchResults Examine.Lucene.Search.LuceneSearchResults.GetEnumerator() -> System.Collections.Generic.IEnumerator! -Examine.Lucene.Search.LuceneSearchResults.LuceneSearchResults(System.Collections.Generic.IReadOnlyCollection! results, int totalItemCount) -> void Examine.Lucene.Search.LuceneSearchResults.TotalItemCount.get -> long Examine.Lucene.Search.MultiSearchContext Examine.Lucene.Search.MultiSearchContext.GetFieldValueType(string! fieldName) -> Examine.Lucene.Indexing.IIndexFieldValueType? @@ -310,12 +288,10 @@ Examine.Lucene.Search.SearchAfterOptions.DocumentId.get -> int Examine.Lucene.Search.SearchAfterOptions.DocumentScore.get -> float Examine.Lucene.Search.SearchAfterOptions.Fields.get -> object![]? Examine.Lucene.Search.SearchAfterOptions.SearchAfterOptions(int documentId, float documentScore, object![]? fields, int shardIndex) -> void -Examine.Lucene.Search.SearchAfterOptions.ShardIndex.get -> int? Examine.Lucene.Search.SearchContext Examine.Lucene.Search.SearchContext.GetFieldValueType(string! fieldName) -> Examine.Lucene.Indexing.IIndexFieldValueType! Examine.Lucene.Search.SearchContext.GetSearcher() -> Examine.Lucene.Search.ISearcherReference! Examine.Lucene.Search.SearchContext.SearchableFields.get -> string![]! -Examine.Lucene.Search.SearchContext.SearchContext(Lucene.Net.Search.SearcherManager! searcherManager, Examine.Lucene.FieldValueTypeCollection! fieldValueTypeCollection) -> void Examine.Lucene.Search.SearcherReference Examine.Lucene.Search.SearcherReference.Dispose() -> void Examine.Lucene.Search.SearcherReference.IndexSearcher.get -> Lucene.Net.Search.IndexSearcher! @@ -330,12 +306,9 @@ Examine.LuceneInfo Examine.StringExtensions override Examine.Lucene.Analyzers.EmailAddressAnalyzer.CreateComponents(string! fieldName, System.IO.TextReader! reader) -> Lucene.Net.Analysis.TokenStreamComponents! override Examine.Lucene.Analyzers.PatternAnalyzer.CreateComponents(string! fieldName, System.IO.TextReader! reader) -> Lucene.Net.Analysis.TokenStreamComponents! -override Examine.Lucene.Directories.FileSystemDirectoryFactory.CreateDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! -override Examine.Lucene.Directories.GenericDirectoryFactory.CreateDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! override Examine.Lucene.Directories.MultiIndexLockFactory.ClearLock(string! lockName) -> void override Examine.Lucene.Directories.MultiIndexLockFactory.MakeLock(string! lockName) -> Lucene.Net.Store.Lock! override Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.CreateDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! -override Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.Dispose(bool disposing) -> void override Examine.Lucene.Indexing.DateTimeType.AddSingleValue(Lucene.Net.Documents.Document! doc, object! value) -> void override Examine.Lucene.Indexing.DateTimeType.GetQuery(string! query) -> Lucene.Net.Search.Query? override Examine.Lucene.Indexing.DateTimeType.GetQuery(System.DateTime? lower, System.DateTime? upper, bool lowerInclusive = true, bool upperInclusive = true) -> Lucene.Net.Search.Query! @@ -375,7 +348,6 @@ override Examine.Lucene.Providers.LuceneIndex.OnIndexingError(Examine.IndexingEr override Examine.Lucene.Providers.LuceneIndex.PerformDeleteFromIndex(System.Collections.Generic.IEnumerable! itemIds, System.Action! onComplete) -> void override Examine.Lucene.Providers.LuceneIndex.PerformIndexItems(System.Collections.Generic.IEnumerable! values, System.Action! onComplete) -> void override Examine.Lucene.Providers.LuceneIndex.Searcher.get -> Examine.ISearcher! -override Examine.Lucene.Providers.LuceneSearcher.GetSearchContext() -> Examine.Lucene.Search.ISearchContext! override Examine.Lucene.Providers.MultiIndexSearcher.GetSearchContext() -> Examine.Lucene.Search.ISearchContext! override Examine.Lucene.Search.CustomMultiFieldQueryParser.GetRangeQuery(string! field, string! part1, string! part2, bool startInclusive, bool endInclusive) -> Lucene.Net.Search.Query! override Examine.Lucene.Search.ExamineMultiFieldQueryParser.GetRangeQuery(string! field, string! part1, string! part2, bool startInclusive, bool endInclusive) -> Lucene.Net.Search.Query! @@ -418,7 +390,6 @@ static Examine.StringExtensions.GenerateHash(this string! str) -> string! static Examine.StringExtensions.GenerateMd5(this string! str) -> string! static Examine.StringExtensions.GenerateSha1Hash(this string! str) -> string! static Examine.StringExtensions.RemoveStopWords(this string! searchText) -> string! -virtual Examine.Lucene.Directories.DirectoryFactoryBase.Dispose(bool disposing) -> void virtual Examine.Lucene.ExamineReplicator.Dispose(bool disposing) -> void virtual Examine.Lucene.Indexing.IndexFieldValueTypeBase.AddValue(Lucene.Net.Documents.Document! doc, object? value) -> void virtual Examine.Lucene.Indexing.IndexFieldValueTypeBase.Analyzer.get -> Lucene.Net.Analysis.Analyzer? @@ -426,10 +397,8 @@ virtual Examine.Lucene.Indexing.IndexFieldValueTypeBase.GetQuery(string! query) virtual Examine.Lucene.Indexing.IndexFieldValueTypeBase.SortableFieldName.get -> string? virtual Examine.Lucene.Providers.LuceneIndex.AddDocument(Lucene.Net.Documents.Document! doc, Examine.ValueSet! valueSet) -> void virtual Examine.Lucene.Providers.LuceneIndex.CreateFieldValueTypes(System.Collections.Generic.IReadOnlyDictionary? indexValueTypesFactory = null) -> Examine.Lucene.FieldValueTypeCollection! -virtual Examine.Lucene.Providers.LuceneIndex.CreateIndexWriter(Lucene.Net.Store.Directory? d) -> Lucene.Net.Index.IndexWriter! virtual Examine.Lucene.Providers.LuceneIndex.Dispose(bool disposing) -> void virtual Examine.Lucene.Providers.LuceneIndex.OnDocumentWriting(Examine.Lucene.DocumentWritingEventArgs! docArgs) -> void -virtual Examine.Lucene.Providers.LuceneSearcher.Dispose(bool disposing) -> void virtual Examine.Lucene.Search.CustomMultiFieldQueryParser.GetFuzzyQueryInternal(string! field, string! termStr, float minSimilarity) -> Lucene.Net.Search.Query! virtual Examine.Lucene.Search.CustomMultiFieldQueryParser.GetProximityQueryInternal(string! field, string! queryText, int slop) -> Lucene.Net.Search.Query! virtual Examine.Lucene.Search.CustomMultiFieldQueryParser.GetWildcardQueryInternal(string! field, string! termStr) -> Lucene.Net.Search.Query! @@ -437,4 +406,3 @@ virtual Examine.Lucene.Search.LuceneSearchQuery.OrderBy(params Examine.Search.So virtual Examine.Lucene.Search.LuceneSearchQuery.OrderByDescending(params Examine.Search.SortableField[]! fields) -> Examine.Search.IBooleanOperation! virtual Examine.Lucene.Search.LuceneSearchQueryBase.GetFieldInternalQuery(string! fieldName, Examine.Search.IExamineValue! fieldValue, bool useQueryParser) -> Lucene.Net.Search.Query? virtual Examine.Lucene.Search.MultiSearchSearcherReference.Dispose(bool disposing) -> void -virtual Examine.Lucene.Search.SearcherReference.Dispose(bool disposing) -> void diff --git a/src/Examine.Lucene/PublicAPI.Unshipped.txt b/src/Examine.Lucene/PublicAPI.Unshipped.txt index 64b7da0e0..2b01b662d 100644 --- a/src/Examine.Lucene/PublicAPI.Unshipped.txt +++ b/src/Examine.Lucene/PublicAPI.Unshipped.txt @@ -1,20 +1,30 @@ #nullable enable +abstract Examine.Lucene.Providers.BaseLuceneSearcher.Dispose() -> void abstract Examine.Lucene.Search.LuceneBooleanOperationBase.WithFacets(System.Action! facets) -> Examine.Search.IQueryExecutor! +Examine.Lucene.Directories.FileSystemDirectoryFactory +Examine.Lucene.Directories.FileSystemDirectoryFactory.CreateTaxonomyDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! +Examine.Lucene.Directories.FileSystemDirectoryFactory.FileSystemDirectoryFactory(System.IO.DirectoryInfo! baseDir, Examine.Lucene.Directories.ILockFactory! lockFactory, Microsoft.Extensions.Options.IOptionsMonitor! indexOptions) -> void +Examine.Lucene.Directories.FileSystemDirectoryFactory.IndexOptions.get -> Microsoft.Extensions.Options.IOptionsMonitor! +Examine.Lucene.Directories.FileSystemDirectoryFactory.LockFactory.get -> Examine.Lucene.Directories.ILockFactory! +Examine.Lucene.Directories.GenericDirectoryFactory.CreateDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! +Examine.Lucene.Directories.GenericDirectoryFactory.CreateTaxonomyDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! Examine.Lucene.Directories.GenericDirectoryFactory.GenericDirectoryFactory(System.Func! factory, System.Func! taxonomyDirectoryFactory) -> void Examine.Lucene.Directories.IDirectoryFactory.CreateTaxonomyDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! -Examine.Lucene.Directories.SyncedTaxonomyFileSystemDirectoryFactory -Examine.Lucene.Directories.SyncedTaxonomyFileSystemDirectoryFactory.SyncedTaxonomyFileSystemDirectoryFactory(System.IO.DirectoryInfo! localDir, System.IO.DirectoryInfo! mainDir, Examine.Lucene.Directories.ILockFactory! lockFactory, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory) -> void +Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo! localDir, System.IO.DirectoryInfo! mainDir, Examine.Lucene.Directories.ILockFactory! lockFactory, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.Extensions.Options.IOptionsMonitor! indexOptions) -> void +Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo! localDir, System.IO.DirectoryInfo! mainDir, Examine.Lucene.Directories.ILockFactory! lockFactory, Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Microsoft.Extensions.Options.IOptionsMonitor! indexOptions, bool tryFixMainIndexIfCorrupt) -> void +Examine.Lucene.Directories.TempEnvFileSystemDirectoryFactory.TempEnvFileSystemDirectoryFactory(Examine.Lucene.Directories.IApplicationIdentifier! applicationIdentifier, Examine.Lucene.Directories.ILockFactory! lockFactory, Microsoft.Extensions.Options.IOptionsMonitor! indexOptions) -> void +Examine.Lucene.ExamineReplicator.ExamineReplicator(Microsoft.Extensions.Logging.ILogger! replicatorLogger, Microsoft.Extensions.Logging.ILogger! clientLogger, Examine.Lucene.Providers.LuceneIndex! sourceIndex, Lucene.Net.Store.Directory! sourceDirectory, Lucene.Net.Store.Directory! destinationDirectory, Lucene.Net.Store.Directory! destinationTaxonomyDirectory, System.IO.DirectoryInfo! tempStorage) -> void Examine.Lucene.ExamineTaxonomyReplicator Examine.Lucene.ExamineTaxonomyReplicator.Dispose() -> void Examine.Lucene.ExamineTaxonomyReplicator.ExamineTaxonomyReplicator(Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, Examine.Lucene.Providers.LuceneIndex! sourceIndex, Lucene.Net.Store.Directory! destinationDirectory, Lucene.Net.Store.Directory! destinationTaxonomyDirectory, System.IO.DirectoryInfo! tempStorage) -> void Examine.Lucene.ExamineTaxonomyReplicator.ReplicateIndex() -> void Examine.Lucene.ExamineTaxonomyReplicator.StartIndexReplicationOnSchedule(int milliseconds) -> void Examine.Lucene.FacetExtensions -Examine.Lucene.Indexing.DateTimeType.DateTimeType(string! fieldName, bool store, bool isFacetable, bool taxonomyIndex, Microsoft.Extensions.Logging.ILoggerFactory! logger, Lucene.Net.Documents.DateResolution resolution) -> void +Examine.Lucene.Indexing.DateTimeType.DateTimeType(string! fieldName, bool isFacetable, bool taxonomyIndex, Microsoft.Extensions.Logging.ILoggerFactory! logger, Lucene.Net.Documents.DateResolution resolution, bool store) -> void Examine.Lucene.Indexing.DateTimeType.IsTaxonomyFaceted.get -> bool Examine.Lucene.Indexing.DoubleType.DoubleType(string! fieldName, bool isFacetable, bool taxonomyIndex, Microsoft.Extensions.Logging.ILoggerFactory! logger, bool store) -> void Examine.Lucene.Indexing.DoubleType.IsTaxonomyFaceted.get -> bool -Examine.Lucene.Indexing.FullTextType.FullTextType(string! fieldName, Microsoft.Extensions.Logging.ILoggerFactory! logger, bool isFacetable, bool taxonomyIndex, bool sortable, Lucene.Net.Analysis.Analyzer! analyzer) -> void +Examine.Lucene.Indexing.FullTextType.FullTextType(string! fieldName, bool isFacetable, bool taxonomyIndex, bool sortable, Microsoft.Extensions.Logging.ILoggerFactory! logger, Lucene.Net.Analysis.Analyzer! analyzer) -> void Examine.Lucene.Indexing.FullTextType.IsTaxonomyFaceted.get -> bool Examine.Lucene.Indexing.IIndexFacetValueType Examine.Lucene.Indexing.IIndexFacetValueType.ExtractFacets(Examine.Lucene.Search.IFacetExtractionContext! facetExtractionContext, Examine.Lucene.Search.IFacetField! field) -> System.Collections.Generic.IEnumerable>! @@ -27,28 +37,32 @@ Examine.Lucene.Indexing.SingleType.IsTaxonomyFaceted.get -> bool Examine.Lucene.Indexing.SingleType.SingleType(string! fieldName, bool isFacetable, bool taxonomyIndex, Microsoft.Extensions.Logging.ILoggerFactory! logger, bool store) -> void Examine.Lucene.LuceneIndexOptions.FacetsConfig.get -> Lucene.Net.Facet.FacetsConfig! Examine.Lucene.LuceneIndexOptions.FacetsConfig.set -> void -Examine.Lucene.LuceneIndexOptions.UseTaxonomyIndex.get -> bool -Examine.Lucene.LuceneIndexOptions.UseTaxonomyIndex.set -> void -Examine.Lucene.Providers.BaseLuceneSearcher.BaseLuceneSearcher(string! name, Lucene.Net.Analysis.Analyzer! analyzer, Lucene.Net.Facet.FacetsConfig! facetsConfig) -> void -Examine.Lucene.Providers.IIndexCommiter -Examine.Lucene.Providers.IIndexCommiter.CommitNow() -> void -Examine.Lucene.Providers.IIndexCommiter.ScheduleCommit() -> void +Examine.Lucene.LuceneMultiSearcherOptions +Examine.Lucene.LuceneMultiSearcherOptions.IndexNames.get -> string![]! +Examine.Lucene.LuceneMultiSearcherOptions.IndexNames.set -> void +Examine.Lucene.LuceneMultiSearcherOptions.LuceneMultiSearcherOptions() -> void +Examine.Lucene.LuceneSearcherOptions +Examine.Lucene.LuceneSearcherOptions.Analyzer.get -> Lucene.Net.Analysis.Analyzer? +Examine.Lucene.LuceneSearcherOptions.Analyzer.set -> void +Examine.Lucene.LuceneSearcherOptions.FacetConfiguration.get -> Lucene.Net.Facet.FacetsConfig? +Examine.Lucene.LuceneSearcherOptions.FacetConfiguration.set -> void +Examine.Lucene.LuceneSearcherOptions.LuceneSearcherOptions() -> void +Examine.Lucene.Providers.BaseLuceneSearcher.BaseLuceneSearcher(string! name, Microsoft.Extensions.Options.IOptionsMonitor! options) -> void +Examine.Lucene.Providers.IIndexCommitter +Examine.Lucene.Providers.IIndexCommitter.CommitError -> System.EventHandler! +Examine.Lucene.Providers.IIndexCommitter.CommitNow() -> void +Examine.Lucene.Providers.IIndexCommitter.Committed -> System.EventHandler? +Examine.Lucene.Providers.IIndexCommitter.ScheduleCommit() -> void Examine.Lucene.Providers.ILuceneTaxonomySearcher Examine.Lucene.Providers.ILuceneTaxonomySearcher.CategoryCount.get -> int Examine.Lucene.Providers.ILuceneTaxonomySearcher.GetOrdinal(string! dim, string![]! path) -> int Examine.Lucene.Providers.ILuceneTaxonomySearcher.GetPath(int ordinal) -> Examine.Search.IFacetLabel! -Examine.Lucene.Providers.LuceneIndex.GetLuceneTaxonomyDirectory() -> Lucene.Net.Store.Directory? -Examine.Lucene.Providers.LuceneIndex.LuceneIndex(Microsoft.Extensions.Logging.ILoggerFactory! loggerFactory, string! name, Microsoft.Extensions.Options.IOptionsMonitor! indexOptions, System.Func! indexCommiterFactory, Lucene.Net.Index.IndexWriter? writer = null) -> void -Examine.Lucene.Providers.LuceneIndex.RaiseIndexCommited(object! sender, System.EventArgs! e) -> void +Examine.Lucene.Providers.LuceneIndex.GetLuceneDirectory() -> Lucene.Net.Store.Directory! +Examine.Lucene.Providers.LuceneIndex.GetLuceneTaxonomyDirectory() -> Lucene.Net.Store.Directory! Examine.Lucene.Providers.LuceneIndex.TaxonomyWriter.get -> Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter! -Examine.Lucene.Providers.LuceneSearcher.LuceneSearcher(string! name, Lucene.Net.Search.SearcherManager! searcherManager, Lucene.Net.Analysis.Analyzer! analyzer, Examine.Lucene.FieldValueTypeCollection! fieldValueTypeCollection, Lucene.Net.Facet.FacetsConfig! facetsConfig) -> void -Examine.Lucene.Providers.LuceneTaxonomySearcher -Examine.Lucene.Providers.LuceneTaxonomySearcher.CategoryCount.get -> int -Examine.Lucene.Providers.LuceneTaxonomySearcher.GetOrdinal(string! dimension, string![]! path) -> int -Examine.Lucene.Providers.LuceneTaxonomySearcher.GetPath(int ordinal) -> Examine.Search.IFacetLabel! -Examine.Lucene.Providers.LuceneTaxonomySearcher.LuceneTaxonomySearcher(string! name, Lucene.Net.Facet.Taxonomy.SearcherTaxonomyManager! searcherManager, Lucene.Net.Analysis.Analyzer! analyzer, Examine.Lucene.FieldValueTypeCollection! fieldValueTypeCollection, Lucene.Net.Facet.FacetsConfig! facetsConfig) -> void -Examine.Lucene.Providers.MultiIndexSearcher.MultiIndexSearcher(string! name, System.Collections.Generic.IEnumerable! indexes, Lucene.Net.Facet.FacetsConfig! facetsConfig, Lucene.Net.Analysis.Analyzer? analyzer = null) -> void -Examine.Lucene.Providers.MultiIndexSearcher.MultiIndexSearcher(string! name, System.Lazy!>! searchers, Lucene.Net.Facet.FacetsConfig! facetsConfig, Lucene.Net.Analysis.Analyzer? analyzer = null) -> void +Examine.Lucene.Providers.MultiIndexSearcher.MultiIndexSearcher(string! name, Microsoft.Extensions.Options.IOptionsMonitor! options, System.Collections.Generic.IEnumerable! indexes) -> void +Examine.Lucene.Providers.MultiIndexSearcher.MultiIndexSearcher(string! name, Microsoft.Extensions.Options.IOptionsMonitor! options, System.Lazy!>! searchers) -> void +Examine.Lucene.Providers.MultiIndexSearcher.Searchers.get -> System.Collections.Generic.IEnumerable! Examine.Lucene.Search.FacetDoubleField Examine.Lucene.Search.FacetDoubleField.DoubleRanges.get -> Examine.Search.DoubleRange[]! Examine.Lucene.Search.FacetDoubleField.ExtractFacets(Examine.Lucene.Search.IFacetExtractionContext! facetExtractionContext) -> System.Collections.Generic.IEnumerable>! @@ -125,41 +139,38 @@ Examine.Lucene.Search.LuceneFacetSamplingQueryOptions.LuceneFacetSamplingQueryOp Examine.Lucene.Search.LuceneFacetSamplingQueryOptions.SampleSize.get -> int Examine.Lucene.Search.LuceneFacetSamplingQueryOptions.Seed.get -> long Examine.Lucene.Search.LuceneQueryOptions.FacetRandomSampling.get -> Examine.Lucene.Search.LuceneFacetSamplingQueryOptions? -Examine.Lucene.Search.LuceneQueryOptions.LuceneQueryOptions(Examine.Lucene.Search.LuceneFacetSamplingQueryOptions? facetSampling, int skip, int? take, Examine.Lucene.Search.SearchAfterOptions? searchAfter, bool trackDocumentScores, bool trackDocumentMaxScore) -> void +Examine.Lucene.Search.LuceneQueryOptions.LuceneQueryOptions(int skip, int take = 100, Examine.Lucene.Search.SearchAfterOptions? searchAfter = null, bool trackDocumentScores = false, bool trackDocumentMaxScore = false, int skipTakeMaxResults = 10000, bool autoCalculateSkipTakeMaxResults = false, Examine.Lucene.Search.LuceneFacetSamplingQueryOptions? facetSampling = null) -> void Examine.Lucene.Search.LuceneSearchQuery.LuceneSearchQuery(Examine.Lucene.Search.ISearchContext! searchContext, string? category, Lucene.Net.Analysis.Analyzer! analyzer, Examine.Lucene.Search.LuceneSearchOptions! searchOptions, Examine.Search.BooleanOperation occurance, Lucene.Net.Facet.FacetsConfig! facetsConfig) -> void Examine.Lucene.Search.LuceneSearchResults.Facets.get -> System.Collections.Generic.IReadOnlyDictionary! Examine.Lucene.Search.LuceneSearchResults.LuceneSearchResults(System.Collections.Generic.IReadOnlyCollection! results, int totalItemCount, System.Collections.Generic.IReadOnlyDictionary! facets, float maxScore, Examine.Lucene.Search.SearchAfterOptions? searchAfterOptions) -> void Examine.Lucene.Search.LuceneSearchResults.MaxScore.get -> float Examine.Lucene.Search.LuceneSearchResults.SearchAfter.get -> Examine.Lucene.Search.SearchAfterOptions? +Examine.Lucene.Search.SearchAfterOptions.ShardIndex.get -> int +Examine.Lucene.Search.SearchContext.SearchContext(Lucene.Net.Search.SearcherManager! searcherManager, Examine.Lucene.FieldValueTypeCollection! fieldValueTypeCollection, bool isNrt) -> void Examine.Lucene.Search.TaxonomySearchContext Examine.Lucene.Search.TaxonomySearchContext.GetFieldValueType(string! fieldName) -> Examine.Lucene.Indexing.IIndexFieldValueType! Examine.Lucene.Search.TaxonomySearchContext.GetSearcher() -> Examine.Lucene.Search.ISearcherReference! Examine.Lucene.Search.TaxonomySearchContext.GetTaxonomyAndSearcher() -> Examine.Lucene.Search.ITaxonomySearcherReference! Examine.Lucene.Search.TaxonomySearchContext.SearchableFields.get -> string![]! -Examine.Lucene.Search.TaxonomySearchContext.TaxonomySearchContext(Lucene.Net.Facet.Taxonomy.SearcherTaxonomyManager! searcherManager, Examine.Lucene.FieldValueTypeCollection! fieldValueTypeCollection) -> void +Examine.Lucene.Search.TaxonomySearchContext.TaxonomySearchContext(Lucene.Net.Facet.Taxonomy.SearcherTaxonomyManager! searcherManager, Examine.Lucene.FieldValueTypeCollection! fieldValueTypeCollection, bool isNrt) -> void Examine.Lucene.Search.TaxonomySearcherReference Examine.Lucene.Search.TaxonomySearcherReference.Dispose() -> void Examine.Lucene.Search.TaxonomySearcherReference.IndexSearcher.get -> Lucene.Net.Search.IndexSearcher! Examine.Lucene.Search.TaxonomySearcherReference.TaxonomyReader.get -> Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyReader! Examine.Lucene.Search.TaxonomySearcherReference.TaxonomySearcherReference(Lucene.Net.Facet.Taxonomy.SearcherTaxonomyManager! searcherManager) -> void -override Examine.Lucene.Directories.FileSystemDirectoryFactory.CreateTaxonomyDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! -override Examine.Lucene.Directories.GenericDirectoryFactory.CreateTaxonomyDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! -override Examine.Lucene.Directories.SyncedTaxonomyFileSystemDirectoryFactory.CreateDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! -override Examine.Lucene.Directories.SyncedTaxonomyFileSystemDirectoryFactory.Dispose(bool disposing) -> void override Examine.Lucene.Indexing.DateTimeType.AddValue(Lucene.Net.Documents.Document! doc, object? value) -> void override Examine.Lucene.Indexing.DoubleType.AddValue(Lucene.Net.Documents.Document! doc, object? value) -> void override Examine.Lucene.Indexing.FullTextType.AddValue(Lucene.Net.Documents.Document! doc, object? value) -> void override Examine.Lucene.Indexing.Int32Type.AddValue(Lucene.Net.Documents.Document! doc, object? value) -> void override Examine.Lucene.Indexing.Int64Type.AddValue(Lucene.Net.Documents.Document! doc, object? value) -> void override Examine.Lucene.Indexing.SingleType.AddValue(Lucene.Net.Documents.Document! doc, object? value) -> void -override Examine.Lucene.Providers.LuceneTaxonomySearcher.Dispose(bool disposing) -> void -override Examine.Lucene.Providers.LuceneTaxonomySearcher.GetSearchContext() -> Examine.Lucene.Search.ISearchContext! +override Examine.Lucene.Providers.MultiIndexSearcher.Dispose() -> void override Examine.Lucene.Search.LuceneBooleanOperation.WithFacets(System.Action! facets) -> Examine.Search.IQueryExecutor! override Examine.Lucene.Search.LuceneFacetOperation.ToString() -> string! static Examine.Lucene.FacetExtensions.GetFacet(this Examine.ISearchResults! searchResults, string! field) -> Examine.Search.IFacetResult? static Examine.Lucene.FacetExtensions.GetFacets(this Examine.ISearchResults! searchResults) -> System.Collections.Generic.IEnumerable! static Examine.Lucene.Search.LuceneSearchExtensions.ExecuteWithLucene(this Examine.Search.IQueryExecutor! queryExecutor, Examine.Search.QueryOptions? options = null) -> Examine.Lucene.Search.ILuceneSearchResults! -virtual Examine.Lucene.Directories.DirectoryFactoryBase.CreateTaxonomyDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! +virtual Examine.Lucene.Directories.FileSystemDirectoryFactory.CreateDirectory(Examine.Lucene.Providers.LuceneIndex! luceneIndex, bool forceUnlock) -> Lucene.Net.Store.Directory! virtual Examine.Lucene.ExamineTaxonomyReplicator.Dispose(bool disposing) -> void virtual Examine.Lucene.Indexing.DateTimeType.ExtractFacets(Examine.Lucene.Search.IFacetExtractionContext! facetExtractionContext, Examine.Lucene.Search.IFacetField! field) -> System.Collections.Generic.IEnumerable>! virtual Examine.Lucene.Indexing.DoubleType.ExtractFacets(Examine.Lucene.Search.IFacetExtractionContext! facetExtractionContext, Examine.Lucene.Search.IFacetField! field) -> System.Collections.Generic.IEnumerable>! @@ -167,11 +178,24 @@ virtual Examine.Lucene.Indexing.FullTextType.ExtractFacets(Examine.Lucene.Search virtual Examine.Lucene.Indexing.Int32Type.ExtractFacets(Examine.Lucene.Search.IFacetExtractionContext! facetExtractionContext, Examine.Lucene.Search.IFacetField! field) -> System.Collections.Generic.IEnumerable>! virtual Examine.Lucene.Indexing.Int64Type.ExtractFacets(Examine.Lucene.Search.IFacetExtractionContext! facetExtractionContext, Examine.Lucene.Search.IFacetField! field) -> System.Collections.Generic.IEnumerable>! virtual Examine.Lucene.Indexing.SingleType.ExtractFacets(Examine.Lucene.Search.IFacetExtractionContext! facetExtractionContext, Examine.Lucene.Search.IFacetField! field) -> System.Collections.Generic.IEnumerable>! -virtual Examine.Lucene.Providers.BaseLuceneSearcher.Dispose() -> void -virtual Examine.Lucene.Providers.BaseLuceneSearcher.Dispose(bool disposing) -> void -virtual Examine.Lucene.Providers.BaseLuceneSearcher.GetDefaultFacetConfig() -> Lucene.Net.Facet.FacetsConfig! -virtual Examine.Lucene.Providers.LuceneIndex.CreateTaxonomyWriter(Lucene.Net.Store.Directory? d) -> Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter! +virtual Examine.Lucene.Providers.LuceneIndex.CreateIndexWriter(Lucene.Net.Store.Directory! d) -> Lucene.Net.Index.IndexWriter! virtual Examine.Lucene.Providers.LuceneIndex.TaxonomySearcher.get -> Examine.Lucene.Providers.ILuceneTaxonomySearcher? -virtual Examine.Lucene.Providers.LuceneTaxonomySearcher.GetTaxonomySearchContext() -> Examine.Lucene.Search.ITaxonomySearchContext! +virtual Examine.Lucene.Providers.LuceneIndex.UpdateLuceneDocument(Lucene.Net.Index.Term! term, Lucene.Net.Documents.Document! doc) -> long? virtual Examine.Lucene.Search.LuceneFacetExtractionContext.GetFacetCounts(string! facetIndexFieldName, bool isTaxonomyIndexed) -> Lucene.Net.Facet.Facets! virtual Examine.Lucene.Search.TaxonomySearcherReference.Dispose(bool disposing) -> void +Examine.Lucene.LuceneIndexOptions.NrtCacheMaxCachedMB.get -> double +Examine.Lucene.LuceneIndexOptions.NrtCacheMaxCachedMB.set -> void +Examine.Lucene.LuceneIndexOptions.NrtCacheMaxMergeSizeMB.get -> double +Examine.Lucene.LuceneIndexOptions.NrtCacheMaxMergeSizeMB.set -> void +Examine.Lucene.LuceneIndexOptions.NrtEnabled.get -> bool +Examine.Lucene.LuceneIndexOptions.NrtEnabled.set -> void +Examine.Lucene.LuceneIndexOptions.NrtTargetMaxStaleSec.get -> double +Examine.Lucene.LuceneIndexOptions.NrtTargetMaxStaleSec.set -> void +Examine.Lucene.LuceneIndexOptions.NrtTargetMinStaleSec.get -> double +Examine.Lucene.LuceneIndexOptions.NrtTargetMinStaleSec.set -> void +Examine.Lucene.Search.LuceneQueryOptions.AutoCalculateSkipTakeMaxResults.get -> bool +Examine.Lucene.Search.LuceneQueryOptions.SearchAfter.get -> Examine.Lucene.Search.SearchAfterOptions +Examine.Lucene.Search.LuceneQueryOptions.SkipTakeMaxResults.get -> int +Examine.Lucene.Search.LuceneSearchResults.SearchAfter.get -> Examine.Lucene.Search.SearchAfterOptions +Examine.Lucene.Search.SearcherReference.SearcherReference() -> void +static Examine.Lucene.Search.LuceneSearchExtensions.ExecuteWithLucene(this Examine.Search.IQueryExecutor queryExecutor, Examine.Search.QueryOptions options = null) -> Examine.Lucene.Search.ILuceneSearchResults diff --git a/src/Examine.Lucene/Search/CustomMultiFieldQueryParser.cs b/src/Examine.Lucene/Search/CustomMultiFieldQueryParser.cs index 0a680ef71..514b20442 100644 --- a/src/Examine.Lucene/Search/CustomMultiFieldQueryParser.cs +++ b/src/Examine.Lucene/Search/CustomMultiFieldQueryParser.cs @@ -1,7 +1,6 @@ using System; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Core; -using Lucene.Net.QueryParsers; using Lucene.Net.QueryParsers.Classic; using Lucene.Net.Search; using Lucene.Net.Util; @@ -14,6 +13,8 @@ namespace Examine.Lucene.Search /// public class CustomMultiFieldQueryParser : MultiFieldQueryParser { + private QueryParser? _keywordAnalyzerQueryParser; + /// public CustomMultiFieldQueryParser(LuceneVersion matchVersion, string[] fields, Analyzer analyzer) : base(matchVersion, fields, analyzer) @@ -21,7 +22,8 @@ public CustomMultiFieldQueryParser(LuceneVersion matchVersion, string[] fields, SearchableFields = fields; } - internal static QueryParser KeywordAnalyzerQueryParser { get; } = new QueryParser(LuceneInfo.CurrentVersion, string.Empty, new KeywordAnalyzer()); + // NOTE: Query parsers are not thread safe so we need to create a new instance here + internal QueryParser KeywordAnalyzerQueryParser => _keywordAnalyzerQueryParser ??= new QueryParser(LuceneInfo.CurrentVersion, string.Empty, new KeywordAnalyzer()); /// /// Fields that are searchable by the query parser diff --git a/src/Examine.Lucene/Search/ILuceneSearchResults.cs b/src/Examine.Lucene/Search/ILuceneSearchResults.cs index 5ff67820d..2cf9d835b 100644 --- a/src/Examine.Lucene/Search/ILuceneSearchResults.cs +++ b/src/Examine.Lucene/Search/ILuceneSearchResults.cs @@ -8,12 +8,12 @@ public interface ILuceneSearchResults : ISearchResults /// /// Options for Searching After. Used for efficent deep paging. /// - SearchAfterOptions? SearchAfter { get; } + public SearchAfterOptions? SearchAfter { get; } /// /// Returns the maximum score value encountered. Note that in case /// scores are not tracked, this returns . /// - float MaxScore { get; } + public float MaxScore { get; } } } diff --git a/src/Examine.Lucene/Search/LuceneQueryOptions.cs b/src/Examine.Lucene/Search/LuceneQueryOptions.cs index db4e5c515..1d3486a4a 100644 --- a/src/Examine.Lucene/Search/LuceneQueryOptions.cs +++ b/src/Examine.Lucene/Search/LuceneQueryOptions.cs @@ -15,34 +15,27 @@ public class LuceneQueryOptions : QueryOptions /// Optional number of result documents to take. /// Optionally skip to results after the results from the previous search execution. Used for efficent deep paging. /// Whether to track the maximum document score. For best performance, if not needed, leave false. + /// When using Skip/Take (not SearchAfter) this will be the maximum data set size that can be paged. + /// If enabled, this will pre-calculate the document count in the index to use for . /// Whether to Track Document Scores. For best performance, if not needed, leave false. - [Obsolete("To remove in Examine 5.0")] -#pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - public LuceneQueryOptions(int skip, int? take = null, SearchAfterOptions? searchAfter = null, bool trackDocumentScores = false, bool trackDocumentMaxScore = false) -#pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads - : base(skip, take) - { - TrackDocumentScores = trackDocumentScores; - TrackDocumentMaxScore = trackDocumentMaxScore; - SearchAfter = searchAfter; - } - - /// - /// Constructor - /// /// Whether to apply Facet sampling to improve performance. If not required, leave null - /// Number of result documents to skip. - /// Optional number of result documents to take. - /// Optionally skip to results after the results from the previous search execution. Used for efficent deep paging. - /// Whether to track the maximum document score. For best performance, if not needed, leave false. - /// Whether to Track Document Scores. For best performance, if not needed, leave false. - public LuceneQueryOptions(LuceneFacetSamplingQueryOptions? facetSampling, int skip, int? take, SearchAfterOptions? searchAfter, bool trackDocumentScores, bool trackDocumentMaxScore) + public LuceneQueryOptions( + int skip, + int take = DefaultMaxResults, + SearchAfterOptions? searchAfter = null, + bool trackDocumentScores = false, + bool trackDocumentMaxScore = false, + int skipTakeMaxResults = AbsoluteMaxResults, + bool autoCalculateSkipTakeMaxResults = false, + LuceneFacetSamplingQueryOptions? facetSampling = null) : base(skip, take) { + SearchAfter = searchAfter; TrackDocumentScores = trackDocumentScores; TrackDocumentMaxScore = trackDocumentMaxScore; - SearchAfter = searchAfter; + SkipTakeMaxResults = skipTakeMaxResults; FacetRandomSampling = facetSampling; + AutoCalculateSkipTakeMaxResults = autoCalculateSkipTakeMaxResults; } /// @@ -56,7 +49,7 @@ public LuceneQueryOptions(LuceneFacetSamplingQueryOptions? facetSampling, int sk public bool TrackDocumentMaxScore { get; } /// - /// Options for Searching After. Used for efficent deep paging. + /// Options for Searching After. Used for efficient deep paging. /// public SearchAfterOptions? SearchAfter { get; } @@ -67,5 +60,23 @@ public LuceneQueryOptions(LuceneFacetSamplingQueryOptions? facetSampling, int sk /// Performance optimization for large sets /// public LuceneFacetSamplingQueryOptions? FacetRandomSampling { get; } + + /// + /// When using Skip/Take (not SearchAfter) this will be the maximum data set size that can be paged. + /// + /// + /// For performance reasons, this should be low. + /// The default is 10k and if larger datasets are required to be paged, + /// this value can be increased but it is recommended to use the SearchAfter feature instead. + /// + public int SkipTakeMaxResults { get; } + + /// + /// If enabled, this will pre-calculate the document count in the index to use for . + /// + /// + /// This will incur a performance hit on each search execution since there will be a query to get the total document count. + /// + public bool AutoCalculateSkipTakeMaxResults { get; } } } diff --git a/src/Examine.Lucene/Search/LuceneSearchExecutor.cs b/src/Examine.Lucene/Search/LuceneSearchExecutor.cs index 053a8e89b..962876488 100644 --- a/src/Examine.Lucene/Search/LuceneSearchExecutor.cs +++ b/src/Examine.Lucene/Search/LuceneSearchExecutor.cs @@ -9,6 +9,7 @@ using Lucene.Net.Facet.Taxonomy; using Lucene.Net.Index; using Lucene.Net.Search; +using Lucene.Net.Util; using LuceneFacetResult = Lucene.Net.Facet.FacetResult; namespace Examine.Lucene.Search @@ -42,21 +43,6 @@ internal LuceneSearchExecutor(QueryOptions? options, Query query, IEnumerable /// Executes a query /// @@ -92,10 +78,6 @@ public ISearchResults Execute() } } - var maxResults = Math.Min((_options.Skip + 1) * _options.Take, MaxDoc); - maxResults = maxResults >= 1 ? maxResults : QueryOptions.DefaultMaxResults; - int numHits = maxResults; - var sortFields = _sortField as SortField[] ?? _sortField.ToArray(); Sort? sort = null; FieldDoc? scoreDocAfter = null; @@ -103,11 +85,20 @@ public ISearchResults Execute() using (var searcher = _searchContext.GetSearcher()) { + var maxSkipTakeDataSetSize = _luceneQueryOptions?.AutoCalculateSkipTakeMaxResults ?? false + ? GetMaxDoc() + : _luceneQueryOptions?.SkipTakeMaxResults ?? QueryOptions.AbsoluteMaxResults; + + var maxResults = Math.Min((_options.Skip + 1) * _options.Take, maxSkipTakeDataSetSize); + maxResults = maxResults >= 1 ? maxResults : QueryOptions.DefaultMaxResults; + int numHits = maxResults; + if (sortFields.Length > 0) { sort = new Sort(sortFields); sort.Rewrite(searcher.IndexSearcher); } + if (_luceneQueryOptions != null && _luceneQueryOptions.SearchAfter != null) { //The document to find results after. @@ -125,7 +116,7 @@ public ISearchResults Execute() if (sortFields.Length > 0) { bool fillFields = true; - topDocsCollector = TopFieldCollector.Create(sort, numHits, scoreDocAfter, fillFields, trackDocScores, trackMaxScore, false); + topDocsCollector = TopFieldCollector.Create(sort!, numHits, scoreDocAfter, fillFields, trackDocScores, trackMaxScore, false); } else { @@ -178,10 +169,16 @@ public ISearchResults Execute() var totalItemCount = topDocs.TotalHits; - var results = new List(topDocs.ScoreDocs.Length); - for (int i = 0; i < topDocs.ScoreDocs.Length; i++) + var results = new List(topDocs.ScoreDocs.Length); + + // TODO: Order by Doc Id for improved perf?? + // Our benchmarks show this is isn't a significant performance improvement, + // but they could be wrong. Sorting by DocId here could only be done if there + // are no sort options. + // See https://cwiki.apache.org/confluence/display/lucene/ImproveSearchingSpeed + foreach (var scoreDoc in topDocs.ScoreDocs) { - var result = GetSearchResult(i, topDocs, searcher.IndexSearcher); + var result = GetSearchResult(scoreDoc, searcher.IndexSearcher); if (result != null) { results.Add(result); @@ -195,6 +192,20 @@ public ISearchResults Execute() } } + /// + /// Used to calculate the total number of documents in the index. + /// + private int GetMaxDoc() + { + if (_maxDoc == null) + { + using var searcher = _searchContext.GetSearcher(); + _maxDoc = searcher.IndexSearcher.IndexReader.MaxDoc; + } + + return _maxDoc.Value; + } + private static FieldDoc GetScoreDocAfter(SearchAfterOptions searchAfterOptions) { FieldDoc scoreDocAfter; @@ -204,9 +215,9 @@ private static FieldDoc GetScoreDocAfter(SearchAfterOptions searchAfterOptions) { searchAfterSortFields = searchAfterOptions.Fields; } - if (searchAfterOptions.ShardIndex != null) + if (searchAfterOptions.ShardIndex >= 0) { - scoreDocAfter = new FieldDoc(searchAfterOptions.DocumentId, searchAfterOptions.DocumentScore, searchAfterSortFields, searchAfterOptions.ShardIndex.Value); + scoreDocAfter = new FieldDoc(searchAfterOptions.DocumentId, searchAfterOptions.DocumentScore, searchAfterSortFields, searchAfterOptions.ShardIndex); } else { @@ -216,7 +227,7 @@ private static FieldDoc GetScoreDocAfter(SearchAfterOptions searchAfterOptions) return scoreDocAfter; } - private static SearchAfterOptions? GetSearchAfterOptions(TopDocs topDocs) + internal static SearchAfterOptions? GetSearchAfterOptions(TopDocs topDocs) { if (topDocs.TotalHits > 0) { @@ -229,6 +240,7 @@ private static FieldDoc GetScoreDocAfter(SearchAfterOptions searchAfterOptions) return new SearchAfterOptions(scoreDoc.Doc, scoreDoc.Score, new object[0], scoreDoc.ShardIndex); } } + return null; } @@ -266,18 +278,8 @@ private IReadOnlyDictionary ExtractFacets(FacetsCollector? return facets; } - private LuceneSearchResult? GetSearchResult(int index, TopDocs topDocs, IndexSearcher luceneSearcher) + private LuceneSearchResult GetSearchResult(ScoreDoc scoreDoc, IndexSearcher luceneSearcher) { - // I have seen IndexOutOfRangeException here which is strange as this is only called in one place - // and from that one place "i" is always less than the size of this collection. - // but we'll error check here anyways - if (topDocs.ScoreDocs.Length < index) - { - return null; - } - - var scoreDoc = topDocs.ScoreDocs[index]; - var docId = scoreDoc.Doc; Document doc; if (_fieldsToLoad != null) @@ -288,6 +290,7 @@ private IReadOnlyDictionary ExtractFacets(FacetsCollector? { doc = luceneSearcher.Doc(docId); } + var score = scoreDoc.Score; var shardIndex = scoreDoc.ShardIndex; var result = CreateSearchResult(doc, score, shardIndex); @@ -301,7 +304,7 @@ private IReadOnlyDictionary ExtractFacets(FacetsCollector? /// The score. /// /// A populated search result object - private LuceneSearchResult CreateSearchResult(Document doc, float score, int shardIndex) + internal static LuceneSearchResult CreateSearchResult(Document doc, float score, int shardIndex) { var id = doc.Get("id"); @@ -312,12 +315,12 @@ private LuceneSearchResult CreateSearchResult(Document doc, float score, int sha var searchResult = new LuceneSearchResult(id, score, () => { - //we can use lucene to find out the fields which have been stored for this particular document + //we can use Lucene to find out the fields which have been stored for this particular document var fields = doc.Fields; var resultVals = new Dictionary>(); - foreach (var field in fields.Cast()) + foreach (var field in fields) { var fieldName = field.Name; var values = doc.GetValues(fieldName); diff --git a/src/Examine.Lucene/Search/LuceneSearchQuery.cs b/src/Examine.Lucene/Search/LuceneSearchQuery.cs index b1a787a34..425fccd90 100644 --- a/src/Examine.Lucene/Search/LuceneSearchQuery.cs +++ b/src/Examine.Lucene/Search/LuceneSearchQuery.cs @@ -191,7 +191,6 @@ internal LuceneBooleanOperationBase RangeQueryInternal(string[] fields, T? mi inner.Add(q, Occur.SHOULD); } } -#if !NETSTANDARD2_0 && !NETSTANDARD2_1 else if(typeof(T) == typeof(DateOnly) && valueType is IIndexRangeValueType dateOnlyType) { var minValueTime = minInclusive ? TimeOnly.MinValue : TimeOnly.MaxValue; @@ -207,7 +206,6 @@ internal LuceneBooleanOperationBase RangeQueryInternal(string[] fields, T? mi inner.Add(q, Occur.SHOULD); } } -#endif else { throw new InvalidOperationException($"Could not perform a range query on the field {f}, it's value type is {valueType?.GetType()}"); @@ -245,13 +243,15 @@ private ISearchResults Search(QueryOptions? options) return EmptySearchResults.Instance; } + // TODO: Use a Filter for category, not a query + // https://cwiki.apache.org/confluence/display/lucene/ImproveSearchingSpeed query = new BooleanQuery { // prefix the category field query as a must { GetFieldInternalQuery(ExamineFieldNames.CategoryFieldName, new ExamineValue(Examineness.Explicit, Category), true), Occur.MUST } }; - // add the ones that we're already existing + // add the ones that were already existing foreach (var c in existingClauses) { query.Add(c); diff --git a/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs b/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs index e8960684e..fb7321a97 100644 --- a/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs +++ b/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs @@ -523,7 +523,7 @@ protected internal LuceneBooleanOperationBase IdInternal(string id, Occur occurr /// /// private Query ParseRawQuery(string rawQuery) - => CustomMultiFieldQueryParser.KeywordAnalyzerQueryParser.Parse(rawQuery); + => _queryParser.KeywordAnalyzerQueryParser.Parse(rawQuery); /// /// Uses a PhraseQuery to build a 'raw/exact' match diff --git a/src/Examine.Lucene/Search/LuceneSearchResults.cs b/src/Examine.Lucene/Search/LuceneSearchResults.cs index 04e3df326..488dcacf5 100644 --- a/src/Examine.Lucene/Search/LuceneSearchResults.cs +++ b/src/Examine.Lucene/Search/LuceneSearchResults.cs @@ -19,16 +19,6 @@ public class LuceneSearchResults : ILuceneSearchResults, IFacetResults private readonly IReadOnlyCollection _results; - /// - [Obsolete("To remove in Examine V5")] - public LuceneSearchResults(IReadOnlyCollection results, int totalItemCount) - { - _results = results; - TotalItemCount = totalItemCount; - MaxScore = float.NaN; - Facets = _noFacets; - } - /// public LuceneSearchResults(IReadOnlyCollection results, int totalItemCount, IReadOnlyDictionary facets, float maxScore, SearchAfterOptions? searchAfterOptions) { diff --git a/src/Examine.Lucene/Search/MultiSearchSearcherReference.cs b/src/Examine.Lucene/Search/MultiSearchSearcherReference.cs index 8d8c35a2d..3900aa657 100644 --- a/src/Examine.Lucene/Search/MultiSearchSearcherReference.cs +++ b/src/Examine.Lucene/Search/MultiSearchSearcherReference.cs @@ -44,7 +44,7 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - foreach(var i in _inner) + foreach (var i in _inner) { i.Dispose(); } diff --git a/src/Examine.Lucene/Search/SearchAfterOptions.cs b/src/Examine.Lucene/Search/SearchAfterOptions.cs index 058555f6e..cb7c34311 100644 --- a/src/Examine.Lucene/Search/SearchAfterOptions.cs +++ b/src/Examine.Lucene/Search/SearchAfterOptions.cs @@ -36,7 +36,7 @@ public SearchAfterOptions(int documentId, float documentScore, object[]? fields, /// /// The index of the shard the doc belongs to /// - public int? ShardIndex { get; } + public int ShardIndex { get; } /// /// Search fields. Should contain null or J2N.Int diff --git a/src/Examine.Lucene/Search/SearchContext.cs b/src/Examine.Lucene/Search/SearchContext.cs index 725e9f62f..2d4e7cff7 100644 --- a/src/Examine.Lucene/Search/SearchContext.cs +++ b/src/Examine.Lucene/Search/SearchContext.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Linq; using Examine.Lucene.Indexing; using Lucene.Net.Index; @@ -9,21 +8,37 @@ namespace Examine.Lucene.Search { /// - public class SearchContext : ISearchContext + public sealed class SearchContext : ISearchContext { private readonly SearcherManager _searcherManager; private readonly FieldValueTypeCollection _fieldValueTypeCollection; + private readonly bool _isNrt; private string[]? _searchableFields; - - /// - public SearchContext(SearcherManager searcherManager, FieldValueTypeCollection fieldValueTypeCollection) + + /// + /// Initializes a new instance of the class. + /// + /// The manager responsible for managing the searcher instances. + /// The collection of field value types used for indexing and searching. + /// Indicates whether the search context is using near real-time indexing. + public SearchContext(SearcherManager searcherManager, FieldValueTypeCollection fieldValueTypeCollection, bool isNrt) { - _searcherManager = searcherManager; + _searcherManager = searcherManager; _fieldValueTypeCollection = fieldValueTypeCollection ?? throw new ArgumentNullException(nameof(fieldValueTypeCollection)); + _isNrt = isNrt; } /// - public ISearcherReference GetSearcher() => new SearcherReference(_searcherManager); + public ISearcherReference GetSearcher() + { + // TODO: Do we want to create a new searcher every time? I think so, but we shouldn't allocate so much + if (!_isNrt) + { + _searcherManager.MaybeRefresh(); + } + + return new SearcherReference(_searcherManager); + } /// public string[] SearchableFields @@ -35,9 +50,10 @@ public string[] SearchableFields // IMPORTANT! Do not resolve the IndexSearcher from the `IndexSearcher` property above since this // will not release it from the searcher manager. When we are collecting fields, we are essentially // performing a 'search'. We must ensure that the underlying reader has the correct reference counts. - IndexSearcher searcher = _searcherManager.Acquire(); + var searcher = _searcherManager.Acquire(); + try - { + { var fields = MultiFields.GetMergedFieldInfos(searcher.IndexReader) .Select(x => x.Name) .ToList(); @@ -62,7 +78,7 @@ public IIndexFieldValueType GetFieldValueType(string fieldName) { //Get the value type for the field, or use the default if not defined return _fieldValueTypeCollection.GetValueType( - fieldName, + fieldName, _fieldValueTypeCollection.ValueTypeFactories.GetRequiredFactory(FieldDefinitionTypes.FullText)); } } diff --git a/src/Examine.Lucene/Search/SearcherReference.cs b/src/Examine.Lucene/Search/SearcherReference.cs index a375da3db..0f89cb845 100644 --- a/src/Examine.Lucene/Search/SearcherReference.cs +++ b/src/Examine.Lucene/Search/SearcherReference.cs @@ -1,58 +1,23 @@ -using System; using Lucene.Net.Search; namespace Examine.Lucene.Search { /// - public class SearcherReference : ISearcherReference + public readonly struct SearcherReference : ISearcherReference { - private bool _disposedValue; private readonly SearcherManager _searcherManager; - private IndexSearcher? _searcher; /// public SearcherReference(SearcherManager searcherManager) { _searcherManager = searcherManager; + IndexSearcher = _searcherManager.Acquire(); } /// - public IndexSearcher IndexSearcher - { - get - { - if (_disposedValue) - { - throw new ObjectDisposedException($"{nameof(SearcherReference)} is disposed"); - } - return _searcher ??= _searcherManager.Acquire(); - } - } + public IndexSearcher IndexSearcher { get; } /// - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - if (_searcher != null) - { - _searcherManager.Release(_searcher); - } - } - - _disposedValue = true; - } - } - - /// - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method -#pragma warning disable IDE0022 // Use expression body for method - Dispose(disposing: true); -#pragma warning restore IDE0022 // Use expression body for method - } + public void Dispose() => _searcherManager.Release(IndexSearcher); } } diff --git a/src/Examine.Lucene/Search/TaxonomySearchContext.cs b/src/Examine.Lucene/Search/TaxonomySearchContext.cs index 56687872a..cbd2593c8 100644 --- a/src/Examine.Lucene/Search/TaxonomySearchContext.cs +++ b/src/Examine.Lucene/Search/TaxonomySearchContext.cs @@ -13,22 +13,28 @@ public class TaxonomySearchContext : ITaxonomySearchContext { private readonly SearcherTaxonomyManager _searcherManager; private readonly FieldValueTypeCollection _fieldValueTypeCollection; + private readonly bool _isNrt; private string[]? _searchableFields; /// /// Constructor /// - /// - /// - /// - public TaxonomySearchContext(SearcherTaxonomyManager searcherManager, FieldValueTypeCollection fieldValueTypeCollection) + public TaxonomySearchContext(SearcherTaxonomyManager searcherManager, FieldValueTypeCollection fieldValueTypeCollection, bool isNrt) { _searcherManager = searcherManager ?? throw new ArgumentNullException(nameof(searcherManager)); _fieldValueTypeCollection = fieldValueTypeCollection ?? throw new ArgumentNullException(nameof(fieldValueTypeCollection)); + _isNrt = isNrt; } /// - public ISearcherReference GetSearcher() => new TaxonomySearcherReference(_searcherManager); + public ISearcherReference GetSearcher() + { + if (!_isNrt) + { + _searcherManager.MaybeRefresh(); + } + return new TaxonomySearcherReference(_searcherManager); + } /// public string[] SearchableFields diff --git a/src/Examine.Lucene/StringExtensions.cs b/src/Examine.Lucene/StringExtensions.cs index a8498dc5d..422a5cfbe 100644 --- a/src/Examine.Lucene/StringExtensions.cs +++ b/src/Examine.Lucene/StringExtensions.cs @@ -1,8 +1,6 @@ using System; -using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Security; using System.Security.Cryptography; using System.Text; using Lucene.Net.Analysis.Standard; @@ -41,7 +39,12 @@ public static string GenerateHash(this string str) /// private static string GenerateHash(this string str, string hashType) { - var hasher = HashAlgorithm.Create(hashType) ?? throw new InvalidOperationException("No hashing type found by name " + hashType); + HashAlgorithm hasher = hashType switch + { + "MD5" => MD5.Create(), + "SHA1" => SHA1.Create(), + _ => throw new NotSupportedException() + }; using (hasher) { @@ -66,17 +69,7 @@ private static string GenerateHash(this string str, string hashType) } } - internal static string EnsureEndsWith(this string input, char value) - { - if (input.EndsWith(value.ToString(CultureInfo.InvariantCulture))) - { - return input; - } - else - { - return input + value; - } - } + internal static string EnsureEndsWith(this string input, char value) => input.EndsWith(value.ToString(CultureInfo.InvariantCulture)) ? input : input + value; internal static string ReplaceNonAlphanumericChars(this string input, string replacement) { @@ -89,24 +82,24 @@ internal static string ReplaceNonAlphanumericChars(this string input, string rep return mName; } - //NOTE: The reason this code is in a separate method is because the Code Analysis barks at us with security concerns for medium trust - // when it is inline in the RemoveStopWords method like it used to be. - - private static bool IsStandardAnalyzerStopWord(string stringToCheck) - { - if (StandardAnalyzer.STOP_WORDS_SET.Contains(stringToCheck.ToLower())) - { - return true; - } - return false; - } + //NOTE: The reason this code is in a separate method is because the Code Analysis barks at us with security concerns for medium trust + // when it is inline in the RemoveStopWords method like it used to be. + + private static bool IsStandardAnalyzerStopWord(string stringToCheck) + { + if (StandardAnalyzer.STOP_WORDS_SET.Contains(stringToCheck.ToLower())) + { + return true; + } + return false; + } /// /// Removes stop words from the text if not contained within a phrase /// /// /// - + public static string RemoveStopWords(this string searchText) { Action removeWords = (str, b) => @@ -115,20 +108,20 @@ public static string RemoveStopWords(this string searchText) var innerBuilder = new StringBuilder(); var searchParts = str.Split(' '); - foreach (var t in searchParts) - { - if (!IsStandardAnalyzerStopWord(t)) - { - innerBuilder.Append(t); + foreach (var t in searchParts) + { + if (!IsStandardAnalyzerStopWord(t)) + { + innerBuilder.Append(t); innerBuilder.Append(" "); - } - } - b.Append(innerBuilder.ToString()); + } + } + b.Append(innerBuilder.ToString()); }; var builder = new StringBuilder(); var carrat = 0; - while(carrat < searchText.Length) + while (carrat < searchText.Length) { var quoteIndex = searchText.IndexOf("\"", carrat); if (quoteIndex >= 0 && carrat == quoteIndex) @@ -156,12 +149,10 @@ public static string RemoveStopWords(this string searchText) { nextCarrat = searchText.Length; } -#pragma warning disable IDE0057 // Use range operator - var terms = searchText.Substring(carrat, nextCarrat - carrat).Trim(); -#pragma warning restore IDE0057 // Use range operator + var terms = searchText[carrat..nextCarrat].Trim(); if (!string.IsNullOrWhiteSpace(terms)) { - removeWords(terms, builder); + removeWords(terms, builder); } carrat = nextCarrat; } diff --git a/src/Examine.Lucene/ValueTypeFactoryCollection.cs b/src/Examine.Lucene/ValueTypeFactoryCollection.cs index dc8f955b0..1de0de850 100644 --- a/src/Examine.Lucene/ValueTypeFactoryCollection.cs +++ b/src/Examine.Lucene/ValueTypeFactoryCollection.cs @@ -37,9 +37,7 @@ public ValueTypeFactoryCollection(IReadOnlyDictionary /// public bool TryGetFactory(string valueTypeName, -#if !NETSTANDARD2_0 [MaybeNullWhen(false)] -#endif out IFieldValueTypeFactory fieldValueTypeFactory) => _valueTypeFactories.TryGetValue(valueTypeName, out fieldValueTypeFactory); @@ -94,27 +92,27 @@ private static IReadOnlyDictionary> G {FieldDefinitionTypes.FacetFloat, name => new SingleType(name, true, false, loggerFactory, true)}, {FieldDefinitionTypes.FacetDouble, name => new DoubleType(name,true, false, loggerFactory, true)}, {FieldDefinitionTypes.FacetLong, name => new Int64Type(name, true, false, loggerFactory, true)}, - {FieldDefinitionTypes.FacetDateTime, name => new DateTimeType(name, true, true, false, loggerFactory, DateResolution.MILLISECOND)}, - {FieldDefinitionTypes.FacetDateYear, name => new DateTimeType(name,true, true, false, loggerFactory, DateResolution.YEAR)}, - {FieldDefinitionTypes.FacetDateMonth, name => new DateTimeType(name,true, true, false, loggerFactory, DateResolution.MONTH)}, - {FieldDefinitionTypes.FacetDateDay, name => new DateTimeType(name, true, true, false, loggerFactory, DateResolution.DAY)}, - {FieldDefinitionTypes.FacetDateHour, name => new DateTimeType(name, true, true, false, loggerFactory, DateResolution.HOUR)}, - {FieldDefinitionTypes.FacetDateMinute, name => new DateTimeType(name, true, true, false, loggerFactory, DateResolution.MINUTE)}, - {FieldDefinitionTypes.FacetFullText, name => new FullTextType(name, loggerFactory, true, false, false, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, - {FieldDefinitionTypes.FacetFullTextSortable, name => new FullTextType(name, loggerFactory, true, false,true, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, + {FieldDefinitionTypes.FacetDateTime, name => new DateTimeType(name, true, false, loggerFactory, DateResolution.MILLISECOND, true)}, + {FieldDefinitionTypes.FacetDateYear, name => new DateTimeType(name, true, false, loggerFactory, DateResolution.YEAR, true)}, + {FieldDefinitionTypes.FacetDateMonth, name => new DateTimeType(name, true, false, loggerFactory, DateResolution.MONTH, true)}, + {FieldDefinitionTypes.FacetDateDay, name => new DateTimeType(name, true, false, loggerFactory, DateResolution.DAY, true)}, + {FieldDefinitionTypes.FacetDateHour, name => new DateTimeType(name, true, false, loggerFactory, DateResolution.HOUR, true)}, + {FieldDefinitionTypes.FacetDateMinute, name => new DateTimeType(name, true, false, loggerFactory, DateResolution.MINUTE, true)}, + {FieldDefinitionTypes.FacetFullText, name => new FullTextType(name, true, false, false, loggerFactory, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, + {FieldDefinitionTypes.FacetFullTextSortable, name => new FullTextType(name, true, false,true, loggerFactory, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, {FieldDefinitionTypes.FacetTaxonomyInteger, name => new Int32Type(name,true,true, loggerFactory, true)}, {FieldDefinitionTypes.FacetTaxonomyFloat, name => new SingleType(name,isFacetable: true, taxonomyIndex: true, loggerFactory, true)}, {FieldDefinitionTypes.FacetTaxonomyDouble, name => new DoubleType(name, true, true, loggerFactory, true)}, {FieldDefinitionTypes.FacetTaxonomyLong, name => new Int64Type(name, isFacetable: true, taxonomyIndex: true, loggerFactory, true)}, - {FieldDefinitionTypes.FacetTaxonomyDateTime, name => new DateTimeType(name,true, true, taxonomyIndex : true, loggerFactory, DateResolution.MILLISECOND)}, - {FieldDefinitionTypes.FacetTaxonomyDateYear, name => new DateTimeType(name, true, true, taxonomyIndex : true, loggerFactory, DateResolution.YEAR)}, - {FieldDefinitionTypes.FacetTaxonomyDateMonth, name => new DateTimeType(name, true, true, taxonomyIndex : true, loggerFactory, DateResolution.MONTH)}, - {FieldDefinitionTypes.FacetTaxonomyDateDay, name => new DateTimeType(name, true, true, taxonomyIndex : true, loggerFactory, DateResolution.DAY)}, - {FieldDefinitionTypes.FacetTaxonomyDateHour, name => new DateTimeType(name, true, isFacetable: true, taxonomyIndex: true, loggerFactory, DateResolution.HOUR)}, - {FieldDefinitionTypes.FacetTaxonomyDateMinute, name => new DateTimeType(name, true, true, taxonomyIndex : true, loggerFactory, DateResolution.MINUTE)}, - {FieldDefinitionTypes.FacetTaxonomyFullText, name => new FullTextType(name, loggerFactory, true, true, false, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, - {FieldDefinitionTypes.FacetTaxonomyFullTextSortable, name => new FullTextType(name, loggerFactory, true, true, true, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, - }; + {FieldDefinitionTypes.FacetTaxonomyDateTime, name => new DateTimeType(name, true, taxonomyIndex : true, loggerFactory, DateResolution.MILLISECOND, true)}, + {FieldDefinitionTypes.FacetTaxonomyDateYear, name => new DateTimeType(name, true, taxonomyIndex : true, loggerFactory, DateResolution.YEAR, true)}, + {FieldDefinitionTypes.FacetTaxonomyDateMonth, name => new DateTimeType(name, true, taxonomyIndex : true, loggerFactory, DateResolution.MONTH, true)}, + {FieldDefinitionTypes.FacetTaxonomyDateDay, name => new DateTimeType(name, true, taxonomyIndex : true, loggerFactory, DateResolution.DAY, true)}, + {FieldDefinitionTypes.FacetTaxonomyDateHour, name => new DateTimeType(name, isFacetable: true, taxonomyIndex: true, loggerFactory, DateResolution.HOUR, true)}, + {FieldDefinitionTypes.FacetTaxonomyDateMinute, name => new DateTimeType(name, true, taxonomyIndex : true, loggerFactory, DateResolution.MINUTE, true)}, + {FieldDefinitionTypes.FacetTaxonomyFullText, name => new FullTextType(name, true, true, false, loggerFactory, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, + {FieldDefinitionTypes.FacetTaxonomyFullTextSortable, name => new FullTextType(name, true, true, true, loggerFactory, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, + }; /// diff --git a/src/Examine.Test/Examine.Core/Options/ConfigureOptionsTests.cs b/src/Examine.Test/Examine.Core/Options/ConfigureOptionsTests.cs index e98e2e22f..a026195b2 100644 --- a/src/Examine.Test/Examine.Core/Options/ConfigureOptionsTests.cs +++ b/src/Examine.Test/Examine.Core/Options/ConfigureOptionsTests.cs @@ -36,7 +36,7 @@ public void Can_Configure_Named_Options() private class MyIndexOptions : IConfigureNamedOptions { - public void Configure(string name, LuceneDirectoryIndexOptions options) + public void Configure(string? name, LuceneDirectoryIndexOptions options) { if (name != "TestIndex") { @@ -74,7 +74,7 @@ public Task StartAsync(CancellationToken cancellationToken) var luceneIndex = index as LuceneIndex; Assert.IsNotNull(luceneIndex); - using (luceneIndex.WithThreadingMode(IndexThreadingMode.Synchronous)) + using (luceneIndex!.WithThreadingMode(IndexThreadingMode.Synchronous)) { luceneIndex.CreateIndex(); luceneIndex.IndexItem( diff --git a/src/Examine.Test/Examine.Core/ValueSetTests.cs b/src/Examine.Test/Examine.Core/ValueSetTests.cs index 3ce5c7a73..886c8d71f 100644 --- a/src/Examine.Test/Examine.Core/ValueSetTests.cs +++ b/src/Examine.Test/Examine.Core/ValueSetTests.cs @@ -65,13 +65,17 @@ public void Given_Enumerable_When_Yielding_ThenConvertsToEnumerable() foreach (var key in input.Keys) { - object[] expected = null; + object[]? expected = null; // ArrayEnumerator does not inherit IEnumerable, so we have to // test both options. if (input[key] is IEnumerable enumerable) + { expected = enumerable.Cast().ToArray(); + } else if (input[key] is Array array) + { expected = array.Cast().ToArray(); + } object[] output = sut.Values[key].ToArray(); @@ -104,17 +108,25 @@ public void Given_SingleAndEnumerableValues_When_Yeilding_ThenConvertsToEnumerab foreach (var key in input.Keys) { - object[] expected = null; + object[]? expected = null; // ArrayEnumerator does not inherit IEnumerable, so we have to // test both options. if (input[key] is string s) + { expected = new object[] { s }; + } else if (input[key] is IEnumerable enumerable) + { expected = enumerable.Cast().ToArray(); + } else if (input[key] is Array array) + { expected = array.Cast().ToArray(); + } else + { expected = new object[] { input[key] }; + } object[] output = sut.Values[key].ToArray(); diff --git a/src/Examine.Test/Examine.Lucene/Analyzers/PatternAnalyzerTests.cs b/src/Examine.Test/Examine.Lucene/Analyzers/PatternAnalyzerTests.cs index 5c138719b..8880c5fcd 100644 --- a/src/Examine.Test/Examine.Lucene/Analyzers/PatternAnalyzerTests.cs +++ b/src/Examine.Test/Examine.Lucene/Analyzers/PatternAnalyzerTests.cs @@ -56,8 +56,10 @@ public void Phone_Number() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, new FieldDefinitionCollection(new FieldDefinition("phone", "phone")), indexValueTypesFactory: valueTypes)) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs new file mode 100644 index 000000000..f2209739a --- /dev/null +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -0,0 +1,349 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using Examine.Lucene; +using Examine.Lucene.Analyzers; +using Examine.Lucene.Directories; +using Examine.Lucene.Providers; +using Lucene.Net.Codecs.Lucene46; +using Lucene.Net.Facet.Taxonomy.Directory; +using Lucene.Net.Index; +using Lucene.Net.Replicator; +using Lucene.Net.Store; +using Microsoft.AspNetCore.DataProtection.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; +using Directory = Lucene.Net.Store.Directory; + +namespace Examine.Test.Examine.Lucene.Directories +{ + [TestFixture] + [NonParallelizable] + public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest + { + private const int ItemCount = 100; + + [TestCase] + public void Given_GenericHostBoot_When_Indexed_Then_ReplicationSucceeds() + { + var appRoot = new DirectoryInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, Guid.NewGuid().ToString())); + var applicationDiscriminator = new MyAppDiscriminator(); + var tempDir = new DirectoryInfo(TempEnvFileSystemDirectoryFactory.GetTempPath( + Mock.Of(x => x.GetApplicationUniqueIdentifier() == applicationDiscriminator.Discriminator))); + + try + { + appRoot.Create(); + + var builder = Host.CreateApplicationBuilder(); + builder.Logging.AddConsole(); + builder.Logging.SetMinimumLevel(LogLevel.Debug); + + var services = builder.Services; + + services.AddSingleton(applicationDiscriminator); + services.AddExamine(appRoot); + services.AddExamineLuceneIndex("MyIndex"); + services.AddExamineLuceneIndex("SyncedIndex"); + + var host = builder.Build(); + + var manager = host.Services.GetRequiredService(); + if (!manager.TryGetIndex("MyIndex", out var i1) || i1 is not LuceneIndex index1) + { + throw new Exception("Index not found"); + } + + if (!manager.TryGetIndex("SyncedIndex", out var i2) || i2 is not LuceneIndex index2) + { + throw new Exception("Index not found"); + } + + var dir1 = (SyncedFileSystemDirectory)index1.GetLuceneDirectory(); + var dir2 = (SyncedFileSystemDirectory)index2.GetLuceneDirectory(); + + var dirInfo1 = ((MMapDirectory)((NRTCachingDirectory)dir1.MainLuceneDirectory).Delegate).Directory; + var dirInfo2 = ((MMapDirectory)((NRTCachingDirectory)dir2.MainLuceneDirectory).Delegate).Directory; + Assert.IsFalse(dirInfo1.Exists); + Assert.IsFalse(dirInfo2.Exists); + + try + { + using (index1.WithThreadingMode(IndexThreadingMode.Synchronous)) + using (index2.WithThreadingMode(IndexThreadingMode.Synchronous)) + { + index1.IndexItem( + new ValueSet(1.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + })); + + index2.IndexItem( + new ValueSet(1.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + })); + } + + // Allow time to complete indexing/file writing/syncing + Thread.Sleep(1000); + + host.Dispose(); + } + finally + { + // Validate that the replication worked and the directory can be read + Assert.IsTrue(dirInfo1.Exists); + Assert.IsTrue(dirInfo2.Exists); + using var mainDir1 = FSDirectory.Open(dirInfo1); + using var mainDir2 = FSDirectory.Open(dirInfo2); + Assert.Greater(mainDir1.ListAll().Length, 1); + Assert.Greater(mainDir2.ListAll().Length, 1); + Assert.IsTrue(DirectoryReader.IndexExists(mainDir1)); + Assert.IsTrue(DirectoryReader.IndexExists(mainDir2)); + } + } + finally + { + appRoot.Delete(true); + tempDir.Delete(true); + } + } + + [TestCase(true, false, true, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] + [TestCase(true, false, false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew)] + [TestCase(true, true, false, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew)] + [TestCase(false, false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] + [Test] + public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedOrOpened( + bool corruptIndex, + bool removeSegments, + bool fixIndex, + Enum expected) + { + var mainPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + + try + { + CreateIndex(mainPath, corruptIndex, removeSegments); + + var syncedDirFactory = new SyncedFileSystemDirectoryFactory( + new DirectoryInfo(tempPath), + new DirectoryInfo(mainPath), + new DefaultLockFactory(), + LoggerFactory, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions()), + fixIndex); + + using var index = new LuceneIndex( + LoggerFactory, + TestIndex.TestIndexName, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + DirectoryFactory = syncedDirFactory + })); + + Directory? dir = null; + try + { + var result = syncedDirFactory.TryCreateDirectory(index, false, out dir); + Assert.IsTrue(result.HasFlag(expected), $"{result} does not have flag {expected}"); + } + finally + { + dir?.Dispose(); + } + } + finally + { + System.IO.Directory.Delete(mainPath, true); + System.IO.Directory.Delete(tempPath, true); + } + } + + [Test] + public void Given_CorruptMainIndex_And_HealthyLocalIndex_When_CreatingDirectory_Then_LocalIndexSyncedToMain() + { + var mainPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + + try + { + // create unhealthy index + CreateIndex(mainPath, true, false); + + // create healthy index + CreateIndex(tempPath, false, false); + + var syncedDirFactory = new SyncedFileSystemDirectoryFactory( + new DirectoryInfo(tempPath), + new DirectoryInfo(mainPath), + new DefaultLockFactory(), + LoggerFactory, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions()), + false); + + using var index = new LuceneIndex( + LoggerFactory, + TestIndex.TestIndexName, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + DirectoryFactory = syncedDirFactory + })); + + Directory? dir = null; + try + { + var result = syncedDirFactory.TryCreateDirectory(index, false, out dir); + Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.SyncedFromLocal)); + } + finally + { + dir?.Dispose(); + } + + // Ensure the docs are there in main + using var mainIndex = new LuceneIndex( + LoggerFactory, + TestIndex.TestIndexName, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + DirectoryFactory = new GenericDirectoryFactory( + _ => FSDirectory.Open(Path.Combine(mainPath, TestIndex.TestIndexName)), + _ => FSDirectory.Open(Path.Combine(mainPath, TestIndex.TestIndexName, "Taxonomy"))), + })); + + var searchResults = mainIndex.Searcher.CreateQuery().All().Execute(); + Assert.AreEqual(ItemCount - 2, searchResults.TotalItemCount); + } + finally + { + System.IO.Directory.Delete(mainPath, true); + System.IO.Directory.Delete(tempPath, true); + } + } + + [Test] + public void Given_CorruptMainIndex_And_CorruptLocalIndex_When_CreatingDirectory_Then_NewIndexesCreatedAndUsable() + { + var mainPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + + try + { + // create unhealthy index + CreateIndex(mainPath, true, false); + + // create unhealthy index + CreateIndex(tempPath, true, false); + + var syncedFactory = new SyncedFileSystemDirectoryFactory( + new DirectoryInfo(tempPath), + new DirectoryInfo(mainPath), + new DefaultLockFactory(), + LoggerFactory, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions()), + false); + + // Ensure the docs are there in main + using var mainIndex = new LuceneIndex( + LoggerFactory, + TestIndex.TestIndexName, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + DirectoryFactory = syncedFactory, + })); + + var searchResults = mainIndex.Searcher.CreateQuery().All().Execute(); + Assert.AreEqual(0, searchResults.TotalItemCount); + } + finally + { + System.IO.Directory.Delete(mainPath, true); + System.IO.Directory.Delete(tempPath, true); + } + } + + private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments) + { + var logger = LoggerFactory.CreateLogger(); + + var indexPath = Path.Combine(rootPath, TestIndex.TestIndexName); + logger.LogInformation($"Creating index at {indexPath} with options: corruptIndex: {corruptIndex}, removeSegments: {removeSegments}"); + + using var luceneDir = FSDirectory.Open(indexPath); + using var luceneTaxonomyDir = FSDirectory.Open(Path.Combine(indexPath, "taxonomy")); + + var taxonomyWriterFactory = new SnapshotDirectoryTaxonomyIndexWriterFactory(); + using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var taxonomyWriter = new DirectoryTaxonomyWriter(taxonomyWriterFactory, luceneTaxonomyDir)) + using (var indexer = GetTestIndex(writer, taxonomyWriterFactory)) + using (indexer.WithThreadingMode(IndexThreadingMode.Synchronous)) + { + var valueSets = new List(); + for (int i = 0; i < ItemCount; i++) + { + valueSets.Add( + new ValueSet(i.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + })); + } + + indexer.IndexItems(valueSets); + + // Now delete some items + indexer.DeleteFromIndex(new[] { "1", "2" }); + + // double ensure we commit here + indexer.IndexWriter.IndexWriter.Commit(); + indexer.IndexWriter.IndexWriter.WaitForMerges(); + } + + + logger.LogInformation("Created index at " + luceneDir.Directory); + Assert.IsTrue(DirectoryReader.IndexExists(luceneDir)); + + if (corruptIndex) + { + CorruptIndex(luceneDir.Directory, removeSegments, logger); + } + } + + private void CorruptIndex(DirectoryInfo dir, bool removeSegments, ILogger logger) + { + // index file extensions (no segments, no gen) + var indexFileExtensions = IndexFileNames.INDEX_EXTENSIONS + .Except(new[] { IndexFileNames.GEN_EXTENSION }) + .ToArray(); + + // Get an index (non segments file) and delete it (corrupt index) + var indexFile = dir.GetFiles() + .Where(x => removeSegments + ? x.Extension.Contains(Lucene46SegmentInfoFormat.SI_EXTENSION, StringComparison.OrdinalIgnoreCase) + : indexFileExtensions.Any(e => IndexFileNames.MatchesExtension(x.Extension, e))) + .First(); + + logger.LogInformation($"Deleting {indexFile.FullName}"); + File.Delete(indexFile.FullName); + } + + private class MyAppDiscriminator : IApplicationDiscriminator + { + public string Discriminator { get; } = Guid.NewGuid().ToString(); + } + } +} diff --git a/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs b/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs index 89c4eca7a..b4fb72467 100644 --- a/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs +++ b/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs @@ -12,8 +12,16 @@ namespace Examine.Test.Examine.Lucene.Sync [TestFixture] public class ExamineReplicatorTests : ExamineBaseTest { - private ILoggerFactory GetLoggerFactory() - => LoggerFactory.Create(x => x.AddConsole().SetMinimumLevel(LogLevel.Debug)); + private readonly ILoggerFactory _loggerFactory = Microsoft.Extensions.Logging.LoggerFactory.Create(x => x.AddConsole().SetMinimumLevel(LogLevel.Debug)); + + private readonly ILogger _replicatorLogger; + private readonly ILogger _clientLogger; + + public ExamineReplicatorTests() + { + _replicatorLogger = _loggerFactory.CreateLogger(); + _clientLogger = _loggerFactory.CreateLogger(); + } [Test] public void GivenAMainIndex_WhenReplicatedLocally_TheLocalIndexIsPopulated() @@ -22,13 +30,15 @@ public void GivenAMainIndex_WhenReplicatedLocally_TheLocalIndexIsPopulated() var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); using (var mainDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) using (var localDir = new RandomIdRAMDirectory()) - using (var mainIndex = GetTestIndex(mainDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineReplicator(GetLoggerFactory(), mainIndex, localDir, tempStorage)) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) + using (var mainIndex = GetTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineReplicator(_replicatorLogger, _clientLogger, mainIndex, mainDir, localDir, localTaxonomyDir, tempStorage)) { mainIndex.CreateIndex(); - mainIndex.IndexItems(mainIndex.AllData()); + mainIndex.IndexItems(TestIndex.AllData()); var mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); Assert.AreEqual(100, mainReader.NumDocs); @@ -42,7 +52,7 @@ public void GivenAMainIndex_WhenReplicatedLocally_TheLocalIndexIsPopulated() // publish on a schedule. replicator.ReplicateIndex(); - using (var localIndex = GetTestIndex(localDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var localIndex = GetTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { var localReader = localIndex.IndexWriter.IndexWriter.GetReader(true); Assert.AreEqual(100, localReader.NumDocs); @@ -57,10 +67,12 @@ public void GivenAnOpenedWriter_WhenReplicationAttempted_ThenAnExceptionIsThrown var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); using (var mainDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) using (var localDir = new RandomIdRAMDirectory()) - using (var mainIndex = GetTestIndex(mainDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineReplicator(GetLoggerFactory(), mainIndex, localDir, tempStorage)) - using (var localIndex = GetTestIndex(localDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) + using (var mainIndex = GetTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineReplicator(_replicatorLogger, _clientLogger, mainIndex, mainDir, localDir, localTaxonomyDir, tempStorage)) + using (var localIndex = GetTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { mainIndex.CreateIndex(); @@ -72,7 +84,7 @@ public void GivenAnOpenedWriter_WhenReplicationAttempted_ThenAnExceptionIsThrown {"item2", new List(new[] {"value2"})} })); - mainIndex.IndexItems(mainIndex.AllData()); + mainIndex.IndexItems(TestIndex.AllData()); Assert.Throws(() => replicator.ReplicateIndex()); } @@ -85,17 +97,19 @@ public void GivenASyncedLocalIndex_WhenTriggered_ThenSyncedBackToMainIndex() var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); using (var mainDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) using (var localDir = new RandomIdRAMDirectory()) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) { - using (var mainIndex = GetTestIndex(mainDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineReplicator(GetLoggerFactory(), mainIndex, localDir, tempStorage)) + using (var mainIndex = GetTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineReplicator(_replicatorLogger, _clientLogger, mainIndex, mainDir, localDir, localTaxonomyDir, tempStorage)) { mainIndex.CreateIndex(); - mainIndex.IndexItems(mainIndex.AllData()); + mainIndex.IndexItems(TestIndex.AllData()); replicator.ReplicateIndex(); } - using (var localIndex = GetTestIndex(localDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var localIndex = GetTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) { localIndex.IndexItem(new ValueSet(9999.ToString(), "content", new Dictionary> @@ -104,13 +118,13 @@ public void GivenASyncedLocalIndex_WhenTriggered_ThenSyncedBackToMainIndex() {"item2", new List(new[] {"value2"})} })); - using (var replicator = new ExamineReplicator(GetLoggerFactory(), localIndex, mainDir, tempStorage)) + using (var replicator = new ExamineReplicator(_replicatorLogger, _clientLogger, localIndex, localDir, mainDir, mainTaxonomyDir, tempStorage)) { // replicate back to main, main index must be closed replicator.ReplicateIndex(); } - using (var mainIndex = GetTestIndex(mainDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var mainIndex = GetTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { var mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); Assert.AreEqual(101, mainReader.NumDocs); @@ -127,28 +141,26 @@ public void GivenASyncedLocalIndex_ThenSyncedBackToMainIndexOnSchedule() var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); using (var mainDir = new RandomIdRAMDirectory()) + using (var mainTaxonomyDir = new RandomIdRAMDirectory()) using (var localDir = new RandomIdRAMDirectory()) + using (var localTaxonomyDir = new RandomIdRAMDirectory()) { - using (var mainIndex = GetTestIndex(mainDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineReplicator(GetLoggerFactory(), mainIndex, localDir, tempStorage)) + using (var mainIndex = GetTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var replicator = new ExamineReplicator(_replicatorLogger, _clientLogger, mainIndex, mainDir, localDir, localTaxonomyDir, tempStorage)) { mainIndex.CreateIndex(); - mainIndex.IndexItems(mainIndex.AllData()); + mainIndex.IndexItems(TestIndex.AllData()); replicator.ReplicateIndex(); } - using (var localIndex = GetTestIndex(localDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) + using (var localIndex = GetTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) { - using (var replicator = new ExamineReplicator( - GetLoggerFactory(), - localIndex, - mainDir, - tempStorage)) + using (var replicator = new ExamineReplicator(_replicatorLogger, _clientLogger, localIndex, localDir, mainDir, mainTaxonomyDir, tempStorage)) { // replicate back to main on schedule replicator.StartIndexReplicationOnSchedule(1000); - for (int i = 0; i < 10; i++) + for (var i = 0; i < 10; i++) { localIndex.IndexItem(new ValueSet(("testing" + i).ToString(), "content", new Dictionary> @@ -164,7 +176,7 @@ public void GivenASyncedLocalIndex_ThenSyncedBackToMainIndexOnSchedule() Thread.Sleep(2000); } - using (var mainIndex = GetTestIndex(mainDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var mainIndex = GetTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { var mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); Assert.AreEqual(110, mainReader.NumDocs); diff --git a/src/Examine.Test/Examine.Lucene/ExamineTaxonomyReplicatorTests.cs b/src/Examine.Test/Examine.Lucene/ExamineTaxonomyReplicatorTests.cs deleted file mode 100644 index b840e94ab..000000000 --- a/src/Examine.Test/Examine.Lucene/ExamineTaxonomyReplicatorTests.cs +++ /dev/null @@ -1,186 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using Examine.Lucene; -using Lucene.Net.Analysis.Standard; -using Lucene.Net.Index; -using Microsoft.Extensions.Logging; -using NUnit.Framework; - -namespace Examine.Test.Examine.Lucene.Sync -{ - [TestFixture] - public class ExamineTaxonomyReplicatorTests : ExamineBaseTest - { - private ILoggerFactory GetLoggerFactory() - => LoggerFactory.Create(x => x.AddConsole().SetMinimumLevel(LogLevel.Debug)); - - [Test] - public void GivenAMainIndex_WhenReplicatedLocally_TheLocalIndexIsPopulated() - { - var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); - var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - - using (var mainDir = new RandomIdRAMDirectory()) - using (var localDir = new RandomIdRAMDirectory()) - using (var mainTaxonomyDir = new RandomIdRAMDirectory()) - using (var localTaxonomyDir = new RandomIdRAMDirectory()) - using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) - { - mainIndex.CreateIndex(); - - mainIndex.IndexItems(mainIndex.AllData()); - - DirectoryReader mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); - Assert.AreEqual(100, mainReader.NumDocs); - - // TODO: Ok so replication CANNOT occur on an open index with an open IndexWriter. - // See this note: https://lucenenet.apache.org/docs/4.8.0-beta00014/api/replicator/Lucene.Net.Replicator.IndexReplicationHandler.html - // "NOTE: This handler assumes that Lucene.Net.Index.IndexWriter is not opened by another process on the index directory. In fact, opening an Lucene.Net.Index.IndexWriter on the same directory to which files are copied can lead to undefined behavior, where some or all the files will be deleted, override other files or simply create a mess. When you replicate an index, it is best if the index is never modified by Lucene.Net.Index.IndexWriter, except the one that is open on the source index, from which you replicate." - // So if we want to replicate, we can sync from Main on startup and ensure that the writer isn't opened until that - // is done (the callback can be used for that). - // If we want to sync back to main, it means we can never open a writer to main, but that might be ok and we - // publish on a schedule. - replicator.ReplicateIndex(); - - using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) - { - DirectoryReader localReader = localIndex.IndexWriter.IndexWriter.GetReader(true); - Assert.AreEqual(100, localReader.NumDocs); - } - } - } - - [Test] - public void GivenAnOpenedWriter_WhenReplicationAttempted_ThenAnExceptionIsThrown() - { - var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); - var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - - using (var mainDir = new RandomIdRAMDirectory()) - using (var localDir = new RandomIdRAMDirectory()) - using (var mainTaxonomyDir = new RandomIdRAMDirectory()) - using (var localTaxonomyDir = new RandomIdRAMDirectory()) - using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) - using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) - { - mainIndex.CreateIndex(); - - // this will open the writer - localIndex.IndexItem(new ValueSet(9999.ToString(), "content", - new Dictionary> - { - {"item1", new List(new[] {"value1"})}, - {"item2", new List(new[] {"value2"})} - })); - - mainIndex.IndexItems(mainIndex.AllData()); - - Assert.Throws(() => replicator.ReplicateIndex()); - } - } - - [Test] - public void GivenASyncedLocalIndex_WhenTriggered_ThenSyncedBackToMainIndex() - { - var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); - var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - - using (var mainDir = new RandomIdRAMDirectory()) - using (var localDir = new RandomIdRAMDirectory()) - using (var mainTaxonomyDir = new RandomIdRAMDirectory()) - using (var localTaxonomyDir = new RandomIdRAMDirectory()) - { - using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) - { - mainIndex.CreateIndex(); - mainIndex.IndexItems(mainIndex.AllData()); - replicator.ReplicateIndex(); - } - - using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - { - localIndex.IndexItem(new ValueSet(9999.ToString(), "content", - new Dictionary> - { - {"item1", new List(new[] {"value1"})}, - {"item2", new List(new[] {"value2"})} - })); - - using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), localIndex, mainDir, mainTaxonomyDir, tempStorage)) - { - // replicate back to main, main index must be closed - replicator.ReplicateIndex(); - } - - using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) - { - DirectoryReader mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); - Assert.AreEqual(101, mainReader.NumDocs); - } - } - } - - } - - [Test] - public void GivenASyncedLocalIndex_ThenSyncedBackToMainIndexOnSchedule() - { - var tempStorage = new System.IO.DirectoryInfo(TestContext.CurrentContext.WorkDirectory); - var indexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - - using (var mainDir = new RandomIdRAMDirectory()) - using (var localDir = new RandomIdRAMDirectory()) - using (var mainTaxonomyDir = new RandomIdRAMDirectory()) - using (var localTaxonomyDir = new RandomIdRAMDirectory()) - { - using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - using (var replicator = new ExamineTaxonomyReplicator(GetLoggerFactory(), mainIndex, localDir, localTaxonomyDir, tempStorage)) - { - mainIndex.CreateIndex(); - mainIndex.IndexItems(mainIndex.AllData()); - replicator.ReplicateIndex(); - } - - using (TestIndex localIndex = GetTaxonomyTestIndex(localDir, localTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), indexDeletionPolicy: indexDeletionPolicy)) - { - using (var replicator = new ExamineTaxonomyReplicator( - GetLoggerFactory(), - localIndex, - mainDir, - mainTaxonomyDir, - tempStorage)) - { - // replicate back to main on schedule - replicator.StartIndexReplicationOnSchedule(1000); - - for (int i = 0; i < 10; i++) - { - localIndex.IndexItem(new ValueSet(("testing" + i).ToString(), "content", - new Dictionary> - { - {"item1", new List(new[] {"value1"})}, - {"item2", new List(new[] {"value2"})} - })); - - Thread.Sleep(500); - } - - // should be plenty to resync everything - Thread.Sleep(2000); - } - - using (TestIndex mainIndex = GetTaxonomyTestIndex(mainDir, mainTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) - { - DirectoryReader mainReader = mainIndex.IndexWriter.IndexWriter.GetReader(true); - Assert.AreEqual(110, mainReader.NumDocs); - } - } - } - - } - } -} diff --git a/src/Examine.Test/Examine.Lucene/Extensions/SpatialSearch.cs b/src/Examine.Test/Examine.Lucene/Extensions/SpatialSearch.cs index c373b5865..a880ce018 100644 --- a/src/Examine.Test/Examine.Lucene/Extensions/SpatialSearch.cs +++ b/src/Examine.Test/Examine.Lucene/Extensions/SpatialSearch.cs @@ -33,7 +33,7 @@ public void Document_Writing_To_Index_Spatial_Data_And_Search_On_100km_Radius_Re // Here's the Java sample code // https://github.com/apache/lucene-solr/blob/branch_4x/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java - SpatialContext ctx = SpatialContext.Geo; + var ctx = SpatialContext.Geo; int maxLevels = 11; //results in sub-meter precision for geohash SpatialPrefixTree grid = new GeohashPrefixTree(ctx, maxLevels); var strategy = new RecursivePrefixTreeStrategy(grid, GeoLocationFieldName); @@ -48,7 +48,7 @@ public void Document_Writing_To_Index_Spatial_Data_And_Search_On_100km_Radius_Re [Test] public void Document_Writing_To_Index_Spatial_Data_And_Search_On_100km_Radius_GetPointVectorStrategy() { - SpatialContext ctx = SpatialContext.Geo; + var ctx = SpatialContext.Geo; var strategy = new PointVectorStrategy(ctx, GeoLocationFieldName); // NOTE: This works without this custom query and only using the filter too @@ -68,13 +68,14 @@ private void RunTest(SpatialContext ctx, SpatialStrategy strategy, Func Indexer_DocumentWriting(args, ctx, strategy); @@ -103,10 +104,10 @@ private void DoSpatialSearch( TestIndex indexer, double searchRadius, string idToMatch, Func createQuery, int lat, int lng) { - var searcher = (LuceneSearcher)indexer.Searcher; + var searcher = (BaseLuceneSearcher)indexer.Searcher; var searchContext = searcher.GetSearchContext(); - using (ISearcherReference searchRef = searchContext.GetSearcher()) + using (var searchRef = searchContext.GetSearcher()) { GetXYFromCoords(lat, lng, out var x, out var y); @@ -159,13 +160,13 @@ private void GetXYFromCoords(double lat, double lng, out double x, out double y) private void Indexer_DocumentWriting(DocumentWritingEventArgs e, SpatialContext ctx, SpatialStrategy strategy) { - double lat = double.Parse(e.ValueSet.Values["lat"].First().ToString()); - double lng = double.Parse(e.ValueSet.Values["lng"].First().ToString()); + var lat = double.Parse(e.ValueSet.Values["lat"][0].ToString()!); + double lng = double.Parse(e.ValueSet.Values["lng"][0].ToString()!); GetXYFromCoords(lat, lng, out var x, out var y); - IPoint geoPoint = ctx.MakePoint(x, y); + var geoPoint = ctx.MakePoint(x, y); - foreach (Field field in strategy.CreateIndexableFields(geoPoint as IShape)) + foreach (var field in strategy.CreateIndexableFields(geoPoint as IShape)) { e.Document.Add(field); } diff --git a/src/Examine.Test/Examine.Lucene/Index/LuceneIndexTests.cs b/src/Examine.Test/Examine.Lucene/Index/LuceneIndexTests.cs index dc9e18306..d1ed56170 100644 --- a/src/Examine.Test/Examine.Lucene/Index/LuceneIndexTests.cs +++ b/src/Examine.Test/Examine.Lucene/Index/LuceneIndexTests.cs @@ -5,19 +5,19 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; +using Examine.Lucene.Analyzers; +using Examine.Lucene.Indexing; +using Examine.Lucene.Providers; +using Examine.Search; using Lucene.Net.Analysis.Standard; +using Lucene.Net.Facet.Taxonomy.Directory; +using Lucene.Net.Index; +using Lucene.Net.Replicator; using Lucene.Net.Store; using NUnit.Framework; -using Lucene.Net.Index; -using Examine.Lucene; -using Examine.Lucene.Providers; -using System.Threading; -using Examine.Lucene.Indexing; -using Examine.Search; -using Examine.Lucene.Analyzers; -using System.Diagnostics; namespace Examine.Test.Examine.Lucene.Index { @@ -31,14 +31,17 @@ public class LuceneIndexTests : ExamineBaseTest [Test] public void Operation_Complete_Executes_For_Single_Item() { - using (var d = new RandomIdRAMDirectory()) - using (var writer = new IndexWriter(d, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) - using (var indexer = GetTestIndex(writer)) + var taxonomyWriterFactory = new SnapshotDirectoryTaxonomyIndexWriterFactory(); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var taxonomyWriter = new DirectoryTaxonomyWriter(taxonomyWriterFactory, luceneTaxonomyDir)) + using (var indexer = GetTestIndex(writer, taxonomyWriterFactory)) { var callCount = 0; var waitHandle = new ManualResetEvent(false); - void OperationComplete(object sender, IndexOperationEventArgs e) + void OperationComplete(object? sender, IndexOperationEventArgs e) { callCount++; //signal that we are done @@ -68,14 +71,17 @@ void OperationComplete(object sender, IndexOperationEventArgs e) [Test] public void Operation_Complete_Executes_For_Multiple_Items() { - using (var d = new RandomIdRAMDirectory()) - using (var writer = new IndexWriter(d, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) - using (var indexer = GetTestIndex(writer)) + var taxonomyWriterFactory = new SnapshotDirectoryTaxonomyIndexWriterFactory(); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var taxonomyWriter = new DirectoryTaxonomyWriter(taxonomyWriterFactory, luceneTaxonomyDir)) + using (var indexer = GetTestIndex(writer, taxonomyWriterFactory)) { var callCount = 0; var waitHandle = new ManualResetEvent(false); - void OperationComplete(object sender, IndexOperationEventArgs e) + void OperationComplete(object? sender, IndexOperationEventArgs e) { callCount++; @@ -92,7 +98,7 @@ void OperationComplete(object sender, IndexOperationEventArgs e) using (indexer.WithThreadingMode(IndexThreadingMode.Asynchronous)) { var tasks = new List(); - for (int i = 0; i < 10; i++) + for (var i = 0; i < 10; i++) { tasks.Add(Task.Run(() => indexer.IndexItem(new ValueSet(i.ToString(), "content", new Dictionary> @@ -114,13 +120,14 @@ void OperationComplete(object sender, IndexOperationEventArgs e) public void Index_Unlocks_When_Disposed() { using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) { Assert.IsFalse(IndexWriter.IsLocked(luceneDir)); - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { indexer.CreateIndex(); - indexer.IndexItems(indexer.AllData()); + indexer.IndexItems(TestIndex.AllData()); Assert.IsTrue(IndexWriter.IsLocked(luceneDir)); } @@ -133,11 +140,12 @@ public void Index_Unlocks_When_Disposed() [Test] public void Rebuild_Index() { - using (var d = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(d, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { indexer.CreateIndex(); - indexer.IndexItems(indexer.AllData()); + indexer.IndexItems(TestIndex.AllData()); var indexWriter = indexer.IndexWriter; var reader = indexWriter.IndexWriter.GetReader(true); @@ -150,7 +158,8 @@ public void Rebuild_Index() public void Index_Exists() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { indexer.EnsureIndex(true); Assert.IsTrue(indexer.IndexExists()); @@ -161,7 +170,8 @@ public void Index_Exists() public void Can_Add_One_Document() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { @@ -182,7 +192,8 @@ public void Can_Add_One_Document() public void Can_Add_Same_Document_Twice_Without_Duplication() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { @@ -206,7 +217,8 @@ public void Can_Add_Same_Document_Twice_Without_Duplication() public void Can_Add_Multiple_Docs() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { @@ -230,7 +242,8 @@ public void Can_Add_Multiple_Docs() public void Can_Delete() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { for (var i = 0; i < 10; i++) { @@ -263,7 +276,8 @@ public void Can_Delete() public void Can_Add_Doc_With_Fields() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { @@ -301,7 +315,8 @@ public void Can_Add_Doc_With_Fields() public void Can_Add_Doc_With_Easy_Fields() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { @@ -331,7 +346,7 @@ void AddData(object sender, IndexingItemEventArgs e, string key, string value) updatedValues[key] = new List() { value }; - e.SetValues(updatedValues.ToDictionary(x=>x.Key, x=>(IEnumerable) x.Value)); + e.SetValues(updatedValues.ToDictionary(x => x.Key, x => (IEnumerable)x.Value)); } void RemoveData(object sender, IndexingItemEventArgs e, string key) @@ -340,15 +355,16 @@ void RemoveData(object sender, IndexingItemEventArgs e, string key) updatedValues.Remove(key); - e.SetValues(updatedValues.ToDictionary(x=>x.Key, x=>(IEnumerable) x.Value)); + e.SetValues(updatedValues.ToDictionary(x => x.Key, x => (IEnumerable)x.Value)); } using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { - indexer.TransformingIndexValues += (sender, e) => AddData(sender, e, "newItem1", "value1"); - indexer.TransformingIndexValues += (sender, e) => RemoveData(sender, e, "item1"); + indexer.TransformingIndexValues += (sender, e) => AddData(sender!, e, "newItem1", "value1"); + indexer.TransformingIndexValues += (sender, e) => RemoveData(sender!, e, "item1"); indexer.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value1" })); @@ -372,7 +388,8 @@ void RemoveData(object sender, IndexingItemEventArgs e, string key) public void Can_Have_Multiple_Values_In_Fields() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { @@ -411,7 +428,8 @@ public void Can_Have_Multiple_Values_In_Fields() public void Can_Update_Document() { using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion))) { @@ -439,8 +457,10 @@ public void Can_Update_Document() public void Number_Field() { using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, new StandardAnalyzer(LuceneInfo.CurrentVersion), new FieldDefinitionCollection(new FieldDefinition("item2", "number")))) { @@ -489,9 +509,12 @@ void WriteLog(string msg) const int ThreadCount = 1000; - using (var d = new RandomIdRAMDirectory()) - using (var writer = new IndexWriter(d, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) - using (var customIndexer = GetTestIndex(writer)) + var taxonomyWriterFactory = new SnapshotDirectoryTaxonomyIndexWriterFactory(); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var taxonomyWriter = new DirectoryTaxonomyWriter(taxonomyWriterFactory, luceneTaxonomyDir)) + using (var customIndexer = GetTestIndex(writer, taxonomyWriterFactory)) using (var customSearcher = (LuceneSearcher)customIndexer.Searcher) { @@ -505,7 +528,7 @@ void WriteLog(string msg) var middleCompletedWaitHandle = new ManualResetEvent(false); var opCompleteCount = 0; - void OperationComplete(object sender, IndexOperationEventArgs e) + void OperationComplete(object? sender, IndexOperationEventArgs e) { Interlocked.Increment(ref opCompleteCount); @@ -530,12 +553,12 @@ void OperationComplete(object sender, IndexOperationEventArgs e) { //get a node from the data repo var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") - .Root + .Root! .Elements() .First(); //get the id for th node we're re-indexing. - var id = (int)node.Attribute("id"); + var id = (int)node.Attribute("id")!; //spawn a bunch of threads to perform some reading var tasks = new List(); @@ -546,7 +569,7 @@ void OperationComplete(object sender, IndexOperationEventArgs e) for (var i = 0; i < ThreadCount; i++) { var indexer = customIndexer; - int docId = i + 1; + var docId = i + 1; tasks.Add(Task.Run(() => { // mimic a slower machine @@ -610,13 +633,16 @@ public void Index_Ensure_No_Duplicates_In_Async() { var rand = new Random(DateTime.Now.Second); - using (var d = new RandomIdRAMDirectory()) - using (var writer = new IndexWriter(d, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) - using (var customIndexer = GetTestIndex(writer)) + var taxonomyWriterFactory = new SnapshotDirectoryTaxonomyIndexWriterFactory(); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var taxonomyWriter = new DirectoryTaxonomyWriter(taxonomyWriterFactory, luceneTaxonomyDir)) + using (var customIndexer = GetTestIndex(writer, taxonomyWriterFactory)) { var waitHandle = new ManualResetEvent(false); - void OperationComplete(object sender, IndexOperationEventArgs e) + void OperationComplete(object? sender, IndexOperationEventArgs e) { //signal that we are done #pragma warning disable IDE0058 // Expression value is never used @@ -638,7 +664,7 @@ void OperationComplete(object sender, IndexOperationEventArgs e) //get a node from the data repo var idQueue = new ConcurrentQueue(Enumerable.Range(1, 3)); var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") - .Root + .Root! .Elements() .First(); @@ -646,14 +672,14 @@ void OperationComplete(object sender, IndexOperationEventArgs e) for (var i = 0; i < idQueue.Count * 20; i++) { //get next id and put it to the back of the list - if (idQueue.TryDequeue(out int docId)) + if (idQueue.TryDequeue(out var docId)) { idQueue.Enqueue(docId); Thread.Sleep(rand.Next(0, 100)); var cloned = new XElement(node); - cloned.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture); + cloned.Attribute("id")!.Value = docId.ToString(CultureInfo.InvariantCulture); Console.WriteLine("Indexing {0}", docId); customIndexer.IndexItems(new[] { cloned.ConvertToValueSet(IndexTypes.Content) }); } @@ -695,11 +721,25 @@ public void Index_Read_And_Write_Ensure_No_Errors_In_Async( // TODO: In this test can we ensure all readers are tracked and closed? // TODO: In the search part, we should be searching in various ways and also with skip - DirectoryInfo temp = null; - global::Lucene.Net.Store.Directory directory; + // capture the original console out + var consoleOut = TestContext.Out; + + void WriteLog(string msg) + { + // reset console out to the orig, this is required because we suppress + // ExecutionContext which is how this is flowed in Nunit so needed when logging + // in OperationComplete + Console.SetOut(consoleOut); + Console.WriteLine(msg); + } + + DirectoryInfo? temp = null; + global::Lucene.Net.Store.Directory luceneDir; + global::Lucene.Net.Store.Directory luceneTaxonomyDir; if (inMemory) { - directory = new RandomIdRAMDirectory(); + luceneDir = new RandomIdRAMDirectory(); + luceneTaxonomyDir = new RandomIdRAMDirectory(); } else { @@ -719,20 +759,29 @@ public void Index_Read_And_Write_Ensure_No_Errors_In_Async( var tempPath = Path.Combine(tempBasePath, Guid.NewGuid().ToString()); System.IO.Directory.CreateDirectory(tempPath); temp = new DirectoryInfo(tempPath); - directory = new SimpleFSDirectory(temp); + luceneDir = FSDirectory.Open(temp); + luceneTaxonomyDir = FSDirectory.Open(Path.Combine(temp.FullName, "taxonomy")); } try { - using (var d = directory) - using (var writer = new IndexWriter(d, - new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) - using (var customIndexer = GetTestIndex(writer)) + var taxonomyWriterFactory = new SnapshotDirectoryTaxonomyIndexWriterFactory(); + using (luceneDir) + using (luceneTaxonomyDir) + using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var taxonomyWriter = new DirectoryTaxonomyWriter(taxonomyWriterFactory, luceneTaxonomyDir)) + using (var customIndexer = GetTestIndex(writer, taxonomyWriterFactory, nrtTargetMaxStaleSec: 1.0, nrtTargetMinStaleSec: 0.1)) using (var customSearcher = (LuceneSearcher)customIndexer.Searcher) using (customIndexer.WithThreadingMode(IndexThreadingMode.Asynchronous)) { + customIndexer.IndexCommitted += (sender, e) => WriteLog("index committed!!!!!!!!!!!!!"); + var waitHandle = new ManualResetEvent(false); - void OperationComplete(object sender, IndexOperationEventArgs e) + // TODO: This seems broken - we wan see many operations complete while we are indexing/searching + // but currently it seems like we are doing all indexing in a single Task which means we only end up + // committing once and then Boom, all searches are available, we want to be able to see search results + // more immediately. + void OperationComplete(object? sender, IndexOperationEventArgs e) { //signal that we are done #pragma warning disable IDE0058 // Expression value is never used @@ -750,7 +799,7 @@ void OperationComplete(object sender, IndexOperationEventArgs e) //get all nodes var nodes = _contentService.GetPublishedContentByXPath("//*[@isDoc]") - .Root + .Root! .Elements() .ToList(); @@ -778,18 +827,18 @@ void doSearch(ISearcher s) for (var counter = 0; counter < searchCountPerThread; counter++) { //get next id and put it to the back of the list - if (idQueue.TryDequeue(out int docId)) + if (idQueue.TryDequeue(out var docId)) { idQueue.Enqueue(docId); var r = s.CreateQuery().Id(docId.ToString()).Execute(); - Console.WriteLine("searching thread: {0}, id: {1}, found: {2}", Thread.CurrentThread.ManagedThreadId, docId, r.Count()); + WriteLog(string.Format("searching thread: {0}, id: {1}, found: {2}", Thread.CurrentThread.ManagedThreadId, docId, r.Count())); Thread.Sleep(searchThreadWait); } } } catch (Exception ex) { - Console.WriteLine("Search ERROR!! {0}", ex); + WriteLog($"Search ERROR!! {ex}"); throw; } } @@ -802,13 +851,13 @@ void doIndex(IIndex ind) for (var i = 0; i < indexCountPerThread; i++) { //get next id and put it to the back of the list - if (idQueue.TryDequeue(out int docId)) + if (idQueue.TryDequeue(out var docId)) { idQueue.Enqueue(docId); var node = getNode(docId - 1); - node.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture); - Console.WriteLine("Indexing {0}", docId); + node.Attribute("id")!.Value = docId.ToString(CultureInfo.InvariantCulture); + WriteLog(string.Format("Indexing {0}", docId)); ind.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Content) }); Thread.Sleep(indexThreadWait); } @@ -816,7 +865,7 @@ void doIndex(IIndex ind) } catch (Exception ex) { - Console.WriteLine("Index ERROR!! {0}", ex); + WriteLog(string.Format("Index ERROR!! {0}", ex)); throw; } } @@ -856,7 +905,7 @@ void doIndex(IIndex ind) customIndexer.WaitForChanges(); var results = customSearcher.CreateQuery().All().Execute(); - Assert.AreEqual(20, results.Count()); + Assert.AreEqual(20, results.Count(), string.Join(", ", results.Select(x => x.Id))); //wait until we are done waitHandle.WaitOne(); diff --git a/src/Examine.Test/Examine.Lucene/Search/AnalyzerTests.cs b/src/Examine.Test/Examine.Lucene/Search/AnalyzerTests.cs index 9b5c8dd53..69ec423fb 100644 --- a/src/Examine.Test/Examine.Lucene/Search/AnalyzerTests.cs +++ b/src/Examine.Test/Examine.Lucene/Search/AnalyzerTests.cs @@ -15,7 +15,8 @@ public void Given_CultureInvariantWhitespaceAnalyzer_When_SearchingBothCharVaria { var analyzer = new CultureInvariantWhitespaceAnalyzer(); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -49,7 +50,8 @@ public void Given_CultureInvariantStandardAnalyzer_When_SearchingBothCharVariant { var analyzer = new CultureInvariantStandardAnalyzer(); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", diff --git a/src/Examine.Test/Examine.Lucene/Search/ConcurrentSearchBenchmarks.cs b/src/Examine.Test/Examine.Lucene/Search/ConcurrentSearchBenchmarks.cs new file mode 100644 index 000000000..e69de29bb diff --git a/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs b/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs index a7e8746d4..25dc167e7 100644 --- a/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs +++ b/src/Examine.Test/Examine.Lucene/Search/FluentApiTests.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Examine.Lucene; using Examine.Lucene.Providers; using Examine.Lucene.Search; @@ -12,8 +14,6 @@ using Lucene.Net.Search; using NUnit.Framework; - - namespace Examine.Test.Examine.Lucene.Search { [TestFixture] @@ -27,6 +27,33 @@ public enum FacetTestType SortedSetFacets } + [Test] + public void Multiple_Searches() + { + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + + using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir, analyzer, nrtEnabled: false)) + { + indexer1.IndexItem(ValueSet.FromObject("1", "content", new { item1 = "value1", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the total absolute darkness." })); + + var searcher = indexer1.Searcher; + + var result = searcher.Search("darkness"); + foreach (var r in result) + { + Console.WriteLine($"Id = {r.Id}, Score = {r.Score}"); + } + + result = searcher.Search("total darkness"); + foreach (var r in result) + { + Console.WriteLine($"Id = {r.Id}, Score = {r.Score}"); + } + } + } + private bool HasFacets(FacetTestType withFacets) => withFacets == FacetTestType.TaxonomyFacets || withFacets == FacetTestType.SortedSetFacets; @@ -35,7 +62,7 @@ public enum FacetTestType [TestCase(FacetTestType.NoFacets)] public void Allow_Leading_Wildcards(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -49,7 +76,7 @@ public void Allow_Leading_Wildcards(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -91,9 +118,10 @@ public void Allow_Leading_Wildcards(FacetTestType withFacets) var facetResults = results1.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results1.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); - Assert.AreEqual(1, facetResults.First().Value); + Assert.AreEqual(2, facetResults!.Count()); + Assert.AreEqual(1, facetResults!.First().Value); } else { @@ -109,7 +137,7 @@ public void Allow_Leading_Wildcards(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void NativeQuery_Single_Word(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -126,7 +154,7 @@ public void NativeQuery_Single_Word(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -153,9 +181,10 @@ public void NativeQuery_Single_Word(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); - Assert.AreEqual(1, facetResults.Last().Value); + Assert.AreEqual(2, facetResults!.Count()); + Assert.AreEqual(1, facetResults!.Last().Value); } else { @@ -171,7 +200,7 @@ public void NativeQuery_Single_Word(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Uppercase_Category(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -188,7 +217,7 @@ public void Uppercase_Category(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -216,8 +245,9 @@ public void Uppercase_Category(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } else { @@ -238,8 +268,10 @@ public void FacetsConfig_SetIndexName_FullText() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection, facetsConfig: @@ -262,8 +294,9 @@ public void FacetsConfig_SetIndexName_FullText() var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } } @@ -277,8 +310,10 @@ public void FacetsConfig_SetIndexName_Long() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection, facetsConfig: facetsConfig)) @@ -308,8 +343,9 @@ public void FacetsConfig_SetIndexName_Long() var facetResults = results.GetFacet("LongValue"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } } @@ -323,8 +359,10 @@ public void FacetsConfig_SetIndexName_Double() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection, facetsConfig: @@ -355,8 +393,9 @@ public void FacetsConfig_SetIndexName_Double() var facetResults = results.GetFacet("DoubleValue"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } } @@ -372,7 +411,7 @@ public void Taxonomy_FacetsConfig_SetIndexName_FullText() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -399,8 +438,9 @@ public void Taxonomy_FacetsConfig_SetIndexName_FullText() var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } } @@ -415,7 +455,7 @@ public void Taxonomy_FacetsConfig_SetIndexName_Long() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -447,8 +487,9 @@ public void Taxonomy_FacetsConfig_SetIndexName_Long() var facetResults = results.GetFacet("LongValue"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } } @@ -464,7 +505,7 @@ public void Taxonomy_FacetsConfig_SetIndexName_Double() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -496,8 +537,9 @@ public void Taxonomy_FacetsConfig_SetIndexName_Double() var facetResults = results.GetFacet("DoubleValue"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } } @@ -506,7 +548,7 @@ public void Taxonomy_FacetsConfig_SetIndexName_Double() [TestCase(FacetTestType.NoFacets)] public void NativeQuery_Phrase(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -523,7 +565,7 @@ public void NativeQuery_Phrase(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -551,8 +593,9 @@ public void NativeQuery_Phrase(FacetTestType withFacets) var facetResults = results.GetFacet("bodyText"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -568,7 +611,7 @@ public void NativeQuery_Phrase(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Managed_Range_Date(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -584,8 +627,10 @@ public void Managed_Range_Date(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { @@ -632,12 +677,13 @@ public void Managed_Range_Date(FacetTestType withFacets) var facetResult = numberSortedResult.GetFacet("created"); + Assert.IsNotNull(facetResult); Assert.AreEqual(2, numberSortedResult.TotalItemCount); - Assert.AreEqual(2, facetResult.Count()); - Assert.AreEqual(1, facetResult.First().Value); - Assert.AreEqual("First days", facetResult.First().Label); - Assert.AreEqual(1, facetResult.Last().Value); - Assert.AreEqual("Last days", facetResult.Last().Label); + Assert.AreEqual(2, facetResult!.Count()); + Assert.AreEqual(1, facetResult!.First().Value); + Assert.AreEqual("First days", facetResult!.First().Label); + Assert.AreEqual(1, facetResult!.Last().Value); + Assert.AreEqual("Last days", facetResult!.Last().Label); } else { @@ -653,7 +699,7 @@ public void Managed_Range_Date(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Managed_Full_Text(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -668,7 +714,7 @@ public void Managed_Full_Text(FacetTestType withFacets) using (var luceneDir1 = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer1 = GetTaxonomyTestIndex( + using (var indexer1 = GetTestIndex( luceneDir1, luceneTaxonomyDir, analyzer, @@ -693,8 +739,9 @@ public void Managed_Full_Text(FacetTestType withFacets) var facetResults = result.GetFacet("item1"); + Assert.IsNotNull(facetResults); Assert.AreEqual(4, result.TotalItemCount); - Assert.AreEqual(4, facetResults.Count()); + Assert.AreEqual(4, facetResults!.Count()); Console.WriteLine("Search 1:"); foreach (var r in result) @@ -708,8 +755,9 @@ public void Managed_Full_Text(FacetTestType withFacets) .Execute(); facetResults = result.GetFacet("item1"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, result.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); Console.WriteLine("Search 2:"); foreach (var r in result) { @@ -743,7 +791,7 @@ public void Managed_Full_Text(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Managed_Full_Text_With_Bool(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -758,7 +806,7 @@ public void Managed_Full_Text_With_Bool(FacetTestType withFacets) using (var luceneDir1 = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer1 = GetTaxonomyTestIndex( + using (var indexer1 = GetTestIndex( luceneDir1, luceneTaxonomyDir, analyzer, @@ -782,8 +830,9 @@ public void Managed_Full_Text_With_Bool(FacetTestType withFacets) var facetResults = result.GetFacet("item1"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, result.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); Console.WriteLine("Search 1:"); foreach (var r in result) { @@ -797,8 +846,9 @@ public void Managed_Full_Text_With_Bool(FacetTestType withFacets) facetResults = result.GetFacet("item1"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, result.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); Console.WriteLine("Search 2:"); foreach (var r in result) { @@ -836,7 +886,7 @@ public void Managed_Full_Text_With_Bool(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Not_Managed_Full_Text(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -851,7 +901,7 @@ public void Not_Managed_Full_Text(FacetTestType withFacets) using (var luceneDir1 = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer1 = GetTaxonomyTestIndex( + using (var indexer1 = GetTestIndex( luceneDir1, luceneTaxonomyDir, analyzer, @@ -878,9 +928,10 @@ public void Not_Managed_Full_Text(FacetTestType withFacets) var facetResults = result.GetFacet("item1"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, result.TotalItemCount); Assert.AreEqual("1", result.ElementAt(0).Id); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); Console.WriteLine("Search 1:"); foreach (var r in result) @@ -909,7 +960,7 @@ public void Not_Managed_Full_Text(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Managed_Range_Int(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -926,14 +977,12 @@ public void Managed_Range_Int(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { - - indexer.IndexItems(new[] { ValueSet.FromObject(123.ToString(), "content", @@ -976,10 +1025,11 @@ public void Managed_Range_Int(FacetTestType withFacets) var facetResults = numberSortedResult.GetFacet("parentID"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, numberSortedResult.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); - Assert.AreEqual(0, facetResults.First(result => result.Label == "120-122").Value); - Assert.AreEqual(2, facetResults.First(result => result.Label == "123-125").Value); + Assert.AreEqual(2, facetResults!.Count()); + Assert.AreEqual(0, facetResults!.First(result => result.Label == "120-122").Value); + Assert.AreEqual(2, facetResults!.First(result => result.Label == "123-125").Value); } else { @@ -995,7 +1045,7 @@ public void Managed_Range_Int(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Legacy_ParentId(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1012,7 +1062,7 @@ public void Legacy_ParentId(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1057,9 +1107,10 @@ public void Legacy_ParentId(FacetTestType withFacets) var facetResults = numberSortedResult.GetFacet("parentID"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, numberSortedResult.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(2, facetResults.Facet("123").Value); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(2, facetResults!.Facet("123")!.Value); } else { @@ -1077,7 +1128,7 @@ public void Legacy_ParentId(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Grouped_Or_Examiness(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1091,7 +1142,7 @@ public void Grouped_Or_Examiness(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1140,14 +1191,15 @@ public void Grouped_Or_Examiness(FacetTestType withFacets) var facetResults = results.GetFacet("nodeTypeAlias"); + Assert.IsNotNull(facetResults); + foreach (var r in results) { Console.WriteLine($"Id = {r.Id}"); } Assert.AreEqual(2, results.TotalItemCount); - - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -1169,7 +1221,8 @@ public void Grouped_Or_Query_Output() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { var searcher = indexer.Searcher; @@ -1219,7 +1272,8 @@ public void Grouped_And_Query_Output() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { var searcher = indexer.Searcher; @@ -1274,7 +1328,8 @@ public void Grouped_Not_Query_Output() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { var searcher = indexer.Searcher; @@ -1321,7 +1376,7 @@ public void Grouped_Not_Query_Output() [TestCase(FacetTestType.NoFacets)] public void Grouped_Not_Single_Field_Single_Value(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1335,7 +1390,7 @@ public void Grouped_Not_Single_Field_Single_Value(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1361,8 +1416,9 @@ public void Grouped_Not_Single_Field_Single_Value(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -1377,7 +1433,7 @@ public void Grouped_Not_Single_Field_Single_Value(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Grouped_Not_Multi_Field_Single_Value(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1390,7 +1446,7 @@ public void Grouped_Not_Multi_Field_Single_Value(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1419,9 +1475,10 @@ public void Grouped_Not_Multi_Field_Single_Value(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(1, facetResults.Facet("my name 2").Value); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(1, facetResults!.Facet("my name 2")!.Value); } else { @@ -1436,7 +1493,7 @@ public void Grouped_Not_Multi_Field_Single_Value(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Grouped_Or_With_Not(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1450,7 +1507,7 @@ public void Grouped_Or_With_Not(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1480,8 +1537,9 @@ public void Grouped_Or_With_Not(FacetTestType withFacets) var facetResults = results.GetFacet("headerText"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Facet("header 2").Value); + Assert.AreEqual(1, facetResults!.Facet("header 2")!.Value); } else { @@ -1496,7 +1554,7 @@ public void Grouped_Or_With_Not(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void And_Grouped_Not_Single_Value(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1510,7 +1568,7 @@ public void And_Grouped_Not_Single_Value(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1538,8 +1596,9 @@ public void And_Grouped_Not_Single_Value(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -1554,7 +1613,7 @@ public void And_Grouped_Not_Single_Value(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void And_Grouped_Not_Multi_Value(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1568,7 +1627,7 @@ public void And_Grouped_Not_Multi_Value(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1595,8 +1654,9 @@ public void And_Grouped_Not_Multi_Value(FacetTestType withFacets) var results = query.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -1611,7 +1671,7 @@ public void And_Grouped_Not_Multi_Value(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void And_Not_Single_Field(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1625,7 +1685,7 @@ public void And_Not_Single_Field(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1670,7 +1730,7 @@ public void And_Not_Single_Field(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void AndNot_Nested(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1684,7 +1744,7 @@ public void AndNot_Nested(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1714,8 +1774,10 @@ public void AndNot_Nested(FacetTestType withFacets) var results = query.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); + + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -1730,7 +1792,7 @@ public void AndNot_Nested(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void And_Not_Added_Later(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1744,7 +1806,7 @@ public void And_Not_Added_Later(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1774,8 +1836,9 @@ public void And_Not_Added_Later(FacetTestType withFacets) var results = query.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -1790,7 +1853,7 @@ public void And_Not_Added_Later(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Not_Range(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1807,7 +1870,7 @@ public void Not_Range(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -1836,9 +1899,11 @@ public void Not_Range(FacetTestType withFacets) .Execute(); var facetResults = results.GetFacet("start"); + + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); Assert.AreEqual(results.First().Id, 1.ToString()); - Assert.AreEqual(0, facetResults.Facet("Label").Value); + Assert.AreEqual(0, facetResults!.Facet("Label")!.Value); } else { @@ -1854,7 +1919,7 @@ public void Not_Range(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Match_By_Path(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -1872,14 +1937,12 @@ public void Match_By_Path(FacetTestType withFacets) using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, fieldDefinitionCollection)) { - - indexer.IndexItems(new[] { new ValueSet(1.ToString(), "content", new Dictionary @@ -1909,8 +1972,10 @@ public void Match_By_Path(FacetTestType withFacets) { var results1 = filter.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults1 = results1.GetFacet("nodeName"); + + Assert.IsNotNull(facetResults1); Assert.AreEqual(1, results1.TotalItemCount); - Assert.AreEqual(1, facetResults1.Count()); + Assert.AreEqual(1, facetResults1!.Count()); } else { @@ -1926,8 +1991,10 @@ public void Match_By_Path(FacetTestType withFacets) { var results2 = exactfilter.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults2 = results2.GetFacet("nodeName"); + + Assert.IsNotNull(facetResults2); Assert.AreEqual(1, results2.TotalItemCount); - Assert.AreEqual(1, facetResults2.Count()); + Assert.AreEqual(1, facetResults2!.Count()); } else { @@ -1945,8 +2012,10 @@ public void Match_By_Path(FacetTestType withFacets) var results5 = nativeFilter.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults5 = results5.GetFacet("nodeName"); + + Assert.IsNotNull(facetResults5); Assert.AreEqual(1, results5.TotalItemCount); - Assert.AreEqual(1, facetResults5.Count()); + Assert.AreEqual(1, facetResults5!.Count()); } else { @@ -1962,8 +2031,10 @@ public void Match_By_Path(FacetTestType withFacets) { var results3 = wildcardfilter.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults3 = results3.GetFacet("nodeName"); + + Assert.IsNotNull(facetResults3); Assert.AreEqual(2, results3.TotalItemCount); - Assert.AreEqual(2, facetResults3.Count()); + Assert.AreEqual(2, facetResults3!.Count()); } else { @@ -1979,6 +2050,7 @@ public void Match_By_Path(FacetTestType withFacets) { var results3 = wildcardfilter.WithFacets(facets => facets.FacetString("nodeName")).Execute(); var facetResults3 = results3.GetFacet("nodeName"); + Assert.AreEqual(0, results3.TotalItemCount); Assert.AreEqual(0, facetResults3?.Count() ?? 0); } @@ -1997,7 +2069,7 @@ public void Match_By_Path(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Find_By_ParentId(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2014,7 +2086,7 @@ public void Find_By_ParentId(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2040,8 +2112,9 @@ public void Find_By_ParentId(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -2057,7 +2130,7 @@ public void Find_By_ParentId(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Find_By_ParentId_Native_Query(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2074,7 +2147,7 @@ public void Find_By_ParentId_Native_Query(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2111,9 +2184,10 @@ public void Find_By_ParentId_Native_Query(FacetTestType withFacets) var facetResults = results.GetFacet("parentID"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(2, facetResults.Facet("1139").Value); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(2, facetResults!.Facet("1139")!.Value); } else { @@ -2129,7 +2203,7 @@ public void Find_By_ParentId_Native_Query(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Find_By_NodeTypeAlias(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2146,7 +2220,7 @@ public void Find_By_NodeTypeAlias(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2191,8 +2265,9 @@ public void Find_By_NodeTypeAlias(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -2208,7 +2283,7 @@ public void Find_By_NodeTypeAlias(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Search_With_Stop_Words(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2222,7 +2297,7 @@ public void Search_With_Stop_Words(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2272,7 +2347,7 @@ public void Search_With_Stop_Words(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Search_Native_Query(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2286,7 +2361,7 @@ public void Search_Native_Query(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2328,9 +2403,10 @@ public void Search_Native_Query(FacetTestType withFacets) var facetResults = results.GetFacet("nodeTypeAlias"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(2, facetResults.Facet("CWS_Home").Value); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(2, facetResults!.Facet("CWS_Home")!.Value); } else { @@ -2348,7 +2424,7 @@ public void Search_Native_Query(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Find_Only_Image_Media(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2362,7 +2438,7 @@ public void Find_Only_Image_Media(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2390,9 +2466,10 @@ public void Find_Only_Image_Media(FacetTestType withFacets) var facetResults = results.GetFacet("nodeTypeAlias"); + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(2, facetResults.Facet("image").Value); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(2, facetResults!.Facet("image")!.Value); } else { @@ -2408,7 +2485,7 @@ public void Find_Only_Image_Media(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Find_Both_Media_And_Content(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2421,7 +2498,7 @@ public void Find_Both_Media_And_Content(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2452,8 +2529,9 @@ public void Find_Both_Media_And_Content(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } else { @@ -2469,7 +2547,7 @@ public void Find_Both_Media_And_Content(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Sort_Result_By_Number_Field(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2486,7 +2564,7 @@ public void Sort_Result_By_Number_Field(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2518,11 +2596,13 @@ public void Sort_Result_By_Number_Field(FacetTestType withFacets) .Execute(); var facetResults = results1.GetFacet("sortOrder"); - var facetReuslts2 = results1.GetFacet("parentID"); + var facetResults2 = results1.GetFacet("parentID"); + Assert.IsNotNull(facetResults); + Assert.IsNotNull(facetResults2); Assert.AreEqual(3, results1.Count()); - Assert.AreEqual(3, facetResults.Count()); - Assert.AreEqual(1, facetReuslts2.Count()); + Assert.AreEqual(3, facetResults!.Count()); + Assert.AreEqual(1, facetResults2!.Count()); var results2 = results1.ToArray(); double currSort = 0; @@ -2553,7 +2633,7 @@ public void Sort_Result_By_Number_Field(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Sort_Result_By_Date_Field(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2570,15 +2650,13 @@ public void Sort_Result_By_Date_Field(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, //Ensure it's set to a date, otherwise it's not sortable fieldDefinitionCollection)) { - - var now = DateTime.Now; indexer.IndexItems(new[] { @@ -2607,11 +2685,13 @@ public void Sort_Result_By_Date_Field(FacetTestType withFacets) .Execute(); var facetResults = results1.GetFacet("updateDate"); - var facetReuslts2 = results1.GetFacet("parentID"); + var facetResults2 = results1.GetFacet("parentID"); + Assert.IsNotNull(facetResults); + Assert.IsNotNull(facetResults2); Assert.AreEqual(3, results1.Count()); - Assert.AreEqual(3, facetResults.Count()); - Assert.AreEqual(1, facetReuslts2.Count()); + Assert.AreEqual(3, facetResults!.Count()); + Assert.AreEqual(1, facetResults2!.Count()); var results2 = results1.ToArray(); double currSort = 0; @@ -2642,7 +2722,7 @@ public void Sort_Result_By_Date_Field(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Sort_Result_By_Single_Field(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2659,7 +2739,7 @@ public void Sort_Result_By_Single_Field(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2695,10 +2775,11 @@ public void Sort_Result_By_Single_Field(FacetTestType withFacets) var facetResults1 = results1.GetFacet("nodeName"); var facetResults2 = results2.GetFacet("nodeName"); + Assert.IsNotNull(facetResults1); + Assert.IsNotNull(facetResults2); Assert.AreNotEqual(results1.First().Id, results2.First().Id); - - Assert.AreEqual(3, facetResults1.Count()); - Assert.AreEqual(3, facetResults2.Count()); + Assert.AreEqual(3, facetResults1!.Count()); + Assert.AreEqual(3, facetResults2!.Count()); } else { @@ -2729,8 +2810,10 @@ public void Sort_Result_By_Double_Fields(string fieldType, SortType sortType, bo // See: https://github.com/Shazwazza/Examine/issues/242 var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, new FieldDefinitionCollection(new FieldDefinition("field1", fieldType)))) { @@ -2780,8 +2863,10 @@ public void Sort_Result_By_Double_Fields(string fieldType, SortType sortType, bo Assert.AreEqual(4.9, double.Parse(results4[1].Values["field1"])); Assert.AreEqual(5.0, double.Parse(results4[0].Values["field1"])); - Assert.AreEqual(6, facetResults1.Count()); - Assert.AreEqual(6, facetResults2.Count()); + Assert.IsNotNull(facetResults1); + Assert.IsNotNull(facetResults2); + Assert.AreEqual(6, facetResults1!.Count()); + Assert.AreEqual(6, facetResults2!.Count()); } else { @@ -2811,7 +2896,7 @@ public void Sort_Result_By_Double_Fields(string fieldType, SortType sortType, bo [TestCase(FacetTestType.NoFacets)] public void Sort_Result_By_Multiple_Fields(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2832,7 +2917,7 @@ public void Sort_Result_By_Multiple_Fields(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2870,8 +2955,10 @@ public void Sort_Result_By_Multiple_Fields(FacetTestType withFacets) Assert.AreEqual("5", results2[4].Id); Assert.AreEqual("4", results2[5].Id); - Assert.AreEqual(6, facetResults.Count()); - Assert.AreEqual(2, facetResults2.Count()); + Assert.IsNotNull(facetResults); + Assert.IsNotNull(facetResults2); + Assert.AreEqual(6, facetResults!.Count()); + Assert.AreEqual(2, facetResults2!.Count()); } else { @@ -2892,7 +2979,7 @@ public void Sort_Result_By_Multiple_Fields(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Standard_Results_Sorted_By_Score(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2906,7 +2993,7 @@ public void Standard_Results_Sorted_By_Score(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -2935,10 +3022,11 @@ public void Standard_Results_Sorted_By_Score(FacetTestType withFacets) var facetResults = results.GetFacet("bodyText"); - Assert.AreEqual(2, facetResults.Count()); + Assert.IsNotNull(facetResults); + Assert.AreEqual(2, facetResults!.Count()); //Assert - for (int i = 0; i < results.TotalItemCount - 1; i++) + for (var i = 0; i < results.TotalItemCount - 1; i++) { var curr = results.ElementAt(i); var next = results.ElementAtOrDefault(i + 1); @@ -2956,7 +3044,7 @@ public void Standard_Results_Sorted_By_Score(FacetTestType withFacets) var results = sc1.Execute(); //Assert - for (int i = 0; i < results.TotalItemCount - 1; i++) + for (var i = 0; i < results.TotalItemCount - 1; i++) { var curr = results.ElementAt(i); var next = results.ElementAtOrDefault(i + 1); @@ -2978,7 +3066,7 @@ public void Standard_Results_Sorted_By_Score(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Skip_Results_Returns_Different_Results(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -2992,7 +3080,7 @@ public void Skip_Results_Returns_Different_Results(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3022,8 +3110,9 @@ public void Skip_Results_Returns_Different_Results(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); //Assert + Assert.IsNotNull(facetResults); Assert.AreNotEqual(results.First(), results.Skip(2).First(), "Third result should be different"); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -3041,7 +3130,7 @@ public void Skip_Results_Returns_Different_Results(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Escaping_Includes_All_Words(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3054,7 +3143,7 @@ public void Escaping_Includes_All_Words(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3085,10 +3174,11 @@ public void Escaping_Includes_All_Words(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults); //Assert //NOTE: The result is 2 because the double space is removed with the analyzer Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -3100,8 +3190,6 @@ public void Escaping_Includes_All_Words(FacetTestType withFacets) Assert.AreEqual(2, results.TotalItemCount); } } - - } [TestCase(FacetTestType.TaxonomyFacets)] @@ -3109,7 +3197,7 @@ public void Escaping_Includes_All_Words(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Grouped_And_Examiness(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3122,7 +3210,7 @@ public void Grouped_And_Examiness(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3158,8 +3246,9 @@ public void Grouped_And_Examiness(FacetTestType withFacets) var facetResults = results.GetFacet("nodeName"); //Assert + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -3177,7 +3266,7 @@ public void Grouped_And_Examiness(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Examiness_Proximity(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3190,7 +3279,7 @@ public void Examiness_Proximity(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3228,8 +3317,9 @@ public void Examiness_Proximity(FacetTestType withFacets) } //Assert + Assert.IsNotNull(facetResults); Assert.AreEqual(3, results.TotalItemCount); - Assert.AreEqual(3, facetResults.Count()); + Assert.AreEqual(3, facetResults!.Count()); } else { @@ -3255,7 +3345,7 @@ public void Examiness_Proximity(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Float_Range_SimpleIndexSet(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3272,7 +3362,7 @@ public void Float_Range_SimpleIndexSet(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3320,12 +3410,14 @@ public void Float_Range_SimpleIndexSet(FacetTestType withFacets) var facetResults2 = results2.GetFacet("SomeFloat"); //Assert + Assert.IsNotNull(facetResults1); + Assert.IsNotNull(facetResults2); Assert.AreEqual(3, results1.TotalItemCount); Assert.AreEqual(1, results2.TotalItemCount); - Assert.AreEqual(2, facetResults1.Facet("1").Value); - Assert.AreEqual(1, facetResults1.Facet("2").Value); - Assert.AreEqual(0, facetResults2.Facet("1").Value); - Assert.AreEqual(1, facetResults2.Facet("2").Value); + Assert.AreEqual(2, facetResults1!.Facet("1")!.Value); + Assert.AreEqual(1, facetResults1!.Facet("2")!.Value); + Assert.AreEqual(0, facetResults2!.Facet("1")!.Value); + Assert.AreEqual(1, facetResults2!.Facet("2")!.Value); } else { @@ -3350,7 +3442,7 @@ public void Float_Range_SimpleIndexSet(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Number_Range_SimpleIndexSet(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3367,7 +3459,7 @@ public void Number_Range_SimpleIndexSet(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3405,10 +3497,12 @@ public void Number_Range_SimpleIndexSet(FacetTestType withFacets) var facetResults2 = results2.GetFacet("SomeNumber"); //Assert + Assert.IsNotNull(facetResults1); + Assert.IsNotNull(facetResults2); Assert.AreEqual(3, results1.TotalItemCount); Assert.AreEqual(1, results2.TotalItemCount); - Assert.AreEqual(1, facetResults1.Count()); - Assert.AreEqual(1, facetResults2.Count()); + Assert.AreEqual(1, facetResults1!.Count()); + Assert.AreEqual(1, facetResults2!.Count()); } else { @@ -3431,7 +3525,7 @@ public void Number_Range_SimpleIndexSet(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Double_Range_SimpleIndexSet(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3448,7 +3542,7 @@ public void Double_Range_SimpleIndexSet(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3495,12 +3589,14 @@ public void Double_Range_SimpleIndexSet(FacetTestType withFacets) var facetResults2 = results2.GetFacet("SomeDouble"); //Assert + Assert.IsNotNull(facetResults1); + Assert.IsNotNull(facetResults2); Assert.AreEqual(3, results1.TotalItemCount); Assert.AreEqual(1, results2.TotalItemCount); - Assert.AreEqual(3, facetResults1.Facet("1").Value); - Assert.AreEqual(0, facetResults1.Facet("2").Value); - Assert.AreEqual(0, facetResults2.Facet("1").Value); - Assert.AreEqual(1, facetResults2.Facet("2").Value); + Assert.AreEqual(3, facetResults1!.Facet("1")!.Value); + Assert.AreEqual(0, facetResults1!.Facet("2")!.Value); + Assert.AreEqual(0, facetResults2!.Facet("1")!.Value); + Assert.AreEqual(1, facetResults2!.Facet("2")!.Value); } else { @@ -3523,7 +3619,7 @@ public void Double_Range_SimpleIndexSet(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Long_Range_SimpleIndexSet(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3540,7 +3636,7 @@ public void Long_Range_SimpleIndexSet(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3586,10 +3682,10 @@ public void Long_Range_SimpleIndexSet(FacetTestType withFacets) //Assert Assert.AreEqual(3, results1.TotalItemCount); Assert.AreEqual(1, results2.TotalItemCount); - Assert.AreEqual(3, facetResults1.Facet("1").Value); - Assert.AreEqual(0, facetResults1.Facet("2").Value); - Assert.AreEqual(0, facetResults2.Facet("1").Value); - Assert.AreEqual(1, facetResults2.Facet("2").Value); + Assert.AreEqual(3, facetResults1!.Facet("1")!.Value); + Assert.AreEqual(0, facetResults1!.Facet("2")!.Value); + Assert.AreEqual(0, facetResults2!.Facet("1")!.Value); + Assert.AreEqual(1, facetResults2!.Facet("2")!.Value); } else { @@ -3614,7 +3710,7 @@ public void Long_Range_SimpleIndexSet(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Date_Range_SimpleIndexSet(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3633,7 +3729,7 @@ public void Date_Range_SimpleIndexSet(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3678,10 +3774,10 @@ public void Date_Range_SimpleIndexSet(FacetTestType withFacets) ////Assert Assert.IsTrue(results.TotalItemCount > 0); Assert.IsTrue(results2.TotalItemCount == 0); - Assert.AreEqual(3, facetResults1.Facet("1").Value); - Assert.AreEqual(0, facetResults1.Facet("2").Value); - Assert.AreEqual(0, facetResults2.Facet("1").Value); - Assert.AreEqual(0, facetResults2.Facet("2").Value); + Assert.AreEqual(3, facetResults1!.Facet("1")!.Value); + Assert.AreEqual(0, facetResults1!.Facet("2")!.Value); + Assert.AreEqual(0, facetResults2!.Facet("1")!.Value); + Assert.AreEqual(0, facetResults2!.Facet("2")!.Value); } else { @@ -3703,7 +3799,7 @@ public void Date_Range_SimpleIndexSet(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Fuzzy_Search(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3717,7 +3813,7 @@ public void Fuzzy_Search(FacetTestType withFacets) var analyzer = new EnglishAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3765,10 +3861,12 @@ public void Fuzzy_Search(FacetTestType withFacets) } ////Assert + Assert.IsNotNull(facetResults1); + Assert.IsNotNull(facetResults2); Assert.AreEqual(2, results.TotalItemCount); Assert.AreEqual(2, results2.TotalItemCount); - Assert.AreEqual(2, facetResults1.Count()); - Assert.AreEqual(2, facetResults2.Count()); + Assert.AreEqual(2, facetResults1!.Count()); + Assert.AreEqual(2, facetResults2!.Count()); } else { @@ -3799,7 +3897,8 @@ public void Execute_With_Take() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -3834,9 +3933,10 @@ public void Execute_With_Take_Max_Results() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { - for (int i = 0; i < 1000; i++) + for (var i = 0; i < 1000; i++) { indexer.IndexItems(new[] { ValueSet.FromObject(i.ToString(), "content", new { Content = "hello world" }) }); } @@ -3863,7 +3963,7 @@ public void Execute_With_Take_Max_Results() [TestCase(FacetTestType.NoFacets)] public void Inner_Or_Query(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3876,7 +3976,7 @@ public void Inner_Or_Query(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3913,8 +4013,9 @@ public void Inner_Or_Query(FacetTestType withFacets) var facetResults = results.GetFacet("Type"); //Assert + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -3932,7 +4033,7 @@ public void Inner_Or_Query(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Inner_And_Query(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -3946,7 +4047,7 @@ public void Inner_And_Query(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -3985,8 +4086,9 @@ public void Inner_And_Query(FacetTestType withFacets) var facetResults = results.GetFacet("Type"); //Assert + Assert.IsNotNull(facetResults); Assert.AreEqual(2, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -4004,7 +4106,7 @@ public void Inner_And_Query(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Inner_Not_Query(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -4018,7 +4120,7 @@ public void Inner_Not_Query(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4057,8 +4159,9 @@ public void Inner_Not_Query(FacetTestType withFacets) var facetResults = results.GetFacet("Type"); //Assert + Assert.IsNotNull(facetResults); Assert.AreEqual(1, results.TotalItemCount); - Assert.AreEqual(1, facetResults.Count()); + Assert.AreEqual(1, facetResults!.Count()); } else { @@ -4076,7 +4179,7 @@ public void Inner_Not_Query(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Complex_Or_Group_Nested_Query(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -4090,7 +4193,7 @@ public void Complex_Or_Group_Nested_Query(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4138,13 +4241,13 @@ public void Complex_Or_Group_Nested_Query(FacetTestType withFacets) var facetResults = results.GetFacet("Type"); //Assert + Assert.IsNotNull(facetResults); foreach (var r in results) { Console.WriteLine($"Result Id: {r.Id}"); } Assert.AreEqual(3, results.TotalItemCount); - - Assert.AreEqual(2, facetResults.Count()); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -4170,7 +4273,7 @@ public void Custom_Lucene_Query_With_Native(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer)) @@ -4200,7 +4303,8 @@ public void Category() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -4227,6 +4331,85 @@ public void Category() } } + [Test] + public void By_Id() + { + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) + { + indexer.IndexItems(new[] { + ValueSet.FromObject(1.ToString(), "content", + new { Content = "hello world", Type = "type1" }), + ValueSet.FromObject(2.ToString(), "content", + new { Content = "hello something or other", Type = "type1" }), + ValueSet.FromObject(3.ToString(), "content", + new { Content = "hello you guys", Type = "type1" }) + }); + + var searcher = indexer.Searcher; + + var query = searcher.CreateQuery().Id(2.ToString()); + Console.WriteLine(query); + + var results = query.Execute(); + + //Assert + Assert.AreEqual(1, results.TotalItemCount); + } + } + + [Ignore("This test needs to be updated to ensure that searching calls GetFieldInternalQuery with useQueryParser = false, see https://github.com/Shazwazza/Examine/issues/335#issuecomment-1834677581")] + [Test] + public void Query_With_Category_Multi_Threaded() + { + var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); + using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) + { + indexer.IndexItems(new[] { + ValueSet.FromObject(1.ToString(), "content", + new { Content = "hello world", Type = "type1" }), + ValueSet.FromObject(2.ToString(), "content", + new { Content = "hello something or other", Type = "type1" }), + ValueSet.FromObject(3.ToString(), "content", + new { Content = "hello you guys", Type = "type3" }), + ValueSet.FromObject(4.ToString(), "media", + new { Content = "hello you cruel world", Type = "type2" }), + ValueSet.FromObject(5.ToString(), "media", + new { Content = "hi there, hello world", Type = "type2" }) + }); + + var searcher = indexer.Searcher; + + var tasks = Enumerable.Range(0, 1) + .Select(x => new Task(() => + { + var criteria = searcher.CreateQuery("content", BooleanOperation.And); + IBooleanOperation examineQuery; + examineQuery = criteria + .GroupedOr(new string[] { "Type" }, "type1", "type2") + .And() + .Field("Content", "hel".MultipleCharacterWildcard()); + + var results = examineQuery.Execute(); + + //Assert + Console.WriteLine(results.TotalItemCount + ", Thread: " + Thread.CurrentThread.ManagedThreadId); + Assert.AreEqual(2, results.TotalItemCount); + })) + .ToArray(); + + Parallel.ForEach(tasks, x => x.Start()); + + Task.WaitAll(tasks); + + Assert.IsTrue(tasks.All(x => x.IsCompletedSuccessfully)); + } + } + //[Test] //public void Wildcard_Results_Sorted_By_Score() //{ @@ -4297,7 +4480,7 @@ public void Category() [TestCase(FacetTestType.NoFacets)] public void Select_Field(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -4311,7 +4494,7 @@ public void Select_Field(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4349,8 +4532,8 @@ public void Select_Field(FacetTestType withFacets) var keys = results.First().Values.Keys.ToArray(); Assert.True(keys.All(x => expectedLoadedFields.Contains(x))); Assert.True(expectedLoadedFields.All(x => keys.Contains(x))); - - Assert.AreEqual(2, facetResults.Count()); + Assert.IsNotNull(facetResults); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -4368,7 +4551,7 @@ public void Select_Field(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Select_Fields(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -4381,7 +4564,7 @@ public void Select_Fields(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4419,8 +4602,8 @@ public void Select_Fields(FacetTestType withFacets) var keys = results.First().Values.Keys.ToArray(); Assert.True(keys.All(x => expectedLoadedFields.Contains(x))); Assert.True(expectedLoadedFields.All(x => keys.Contains(x))); - - Assert.AreEqual(2, facetResults.Count()); + Assert.IsNotNull(facetResults); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -4439,7 +4622,7 @@ public void Select_Fields(FacetTestType withFacets) [TestCase(FacetTestType.NoFacets)] public void Select_Fields_HashSet(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -4452,7 +4635,7 @@ public void Select_Fields_HashSet(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4490,8 +4673,8 @@ public void Select_Fields_HashSet(FacetTestType withFacets) var keys = results.First().Values.Keys.ToArray(); Assert.True(keys.All(x => expectedLoadedFields.Contains(x))); Assert.True(expectedLoadedFields.All(x => keys.Contains(x))); - - Assert.AreEqual(2, facetResults.Count()); + Assert.IsNotNull(facetResults); + Assert.AreEqual(2, facetResults!.Count()); } else { @@ -4509,7 +4692,8 @@ public void Select_Fields_Native_Query() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { @@ -4547,7 +4731,8 @@ public void Can_Skip() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -4579,12 +4764,13 @@ public void Can_Skip() } } + // TODO: Redo this with the updated code Paging_With_Skip_Take from v3 [TestCase(FacetTestType.TaxonomyFacets)] [TestCase(FacetTestType.SortedSetFacets)] [TestCase(FacetTestType.NoFacets)] public void Paging_With_Skip_Take(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -4597,7 +4783,7 @@ public void Paging_With_Skip_Take(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4623,8 +4809,8 @@ public void Paging_With_Skip_Take(FacetTestType withFacets) //Arrange var sc = searcher.CreateQuery("content").Field("writerName", "administrator"); - int pageIndex = 0; - int pageSize = 2; + var pageIndex = 0; + var pageSize = 2; //Act if (HasFacets(withFacets)) @@ -4634,8 +4820,9 @@ public void Paging_With_Skip_Take(FacetTestType withFacets) .Execute(QueryOptions.SkipTake(pageIndex * pageSize, pageSize)); Assert.AreEqual(2, results.Count()); var facetResults = results.GetFacet("writerName"); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(5, facetResults.Facet("administrator").Value); + Assert.IsNotNull(facetResults); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(5, facetResults!.Facet("administrator")!.Value); pageIndex++; @@ -4643,8 +4830,9 @@ public void Paging_With_Skip_Take(FacetTestType withFacets) .Execute(QueryOptions.SkipTake(pageIndex * pageSize, pageSize)); Assert.AreEqual(2, results.Count()); facetResults = results.GetFacet("writerName"); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(5, facetResults.Facet("administrator").Value); + Assert.IsNotNull(facetResults); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(5, facetResults!.Facet("administrator")!.Value); pageIndex++; @@ -4652,8 +4840,9 @@ public void Paging_With_Skip_Take(FacetTestType withFacets) .Execute(QueryOptions.SkipTake(pageIndex * pageSize, pageSize)); Assert.AreEqual(1, results.Count()); facetResults = results.GetFacet("writerName"); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(5, facetResults.Facet("administrator").Value); + Assert.IsNotNull(facetResults); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(5, facetResults!.Facet("administrator")!.Value); pageIndex++; @@ -4661,8 +4850,9 @@ public void Paging_With_Skip_Take(FacetTestType withFacets) .Execute(QueryOptions.SkipTake(pageIndex * pageSize, pageSize)); Assert.AreEqual(0, results.Count()); facetResults = results.GetFacet("writerName"); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(5, facetResults.Facet("administrator").Value); + Assert.IsNotNull(facetResults); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(5, facetResults!.Facet("administrator")!.Value); } else { @@ -4730,7 +4920,7 @@ public void Given_SkipTake_Returns_ExpectedTotals(int skip, int take, int expect var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4755,10 +4945,11 @@ public void Given_SkipTake_Returns_ExpectedTotals(int skip, int take, int expect var facetResults = results.GetFacet("nodeName"); + Assert.IsNotNull(facetResults, "Facet results should not be null"); Assert.AreEqual(indexSize, results.TotalItemCount); Assert.AreEqual(expectedResults, results.Count()); - Assert.AreEqual(1, facetResults.Count()); - Assert.AreEqual(5, facetResults.Facet("umbraco").Value); + Assert.AreEqual(1, facetResults!.Count()); + Assert.AreEqual(5, facetResults!.Facet("umbraco")!.Value); } else { @@ -4775,7 +4966,8 @@ public void SearchAfter_Sorted_Results_Returns_Different_Results() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -4809,10 +5001,11 @@ public void SearchAfter_Sorted_Results_Returns_Different_Results() Assert.IsTrue(luceneResults1List.Any(x => x.Id == "2")); // Second query result continues after result 1 (zero indexed), Takes 1, should not include any of the results before or include the SearchAfter docid / scoreid - var searchAfter = new SearchAfterOptions(luceneResults.SearchAfter.DocumentId, - luceneResults.SearchAfter.DocumentScore, - luceneResults.SearchAfter.Fields, - luceneResults.SearchAfter.ShardIndex.Value); + var searchAfter = new SearchAfterOptions( + luceneResults.SearchAfter!.DocumentId, + luceneResults.SearchAfter!.DocumentScore, + luceneResults.SearchAfter!.Fields, + luceneResults.SearchAfter!.ShardIndex); var luceneOptions2 = new LuceneQueryOptions(0, 1, searchAfter); var luceneResults2 = sc.ExecuteWithLucene(luceneOptions2); var luceneResults2List = luceneResults2.ToList(); @@ -4822,7 +5015,7 @@ public void SearchAfter_Sorted_Results_Returns_Different_Results() Assert.IsFalse(luceneResults2List.Any(x => luceneResults.ToList().Any(y => y.Id == x.Id)), "Results should not overlap"); // Third query result continues after result 2 (zero indexed), Takes 1 - var searchAfter2 = new SearchAfterOptions(luceneResults2.SearchAfter.DocumentId, luceneResults2.SearchAfter.DocumentScore, luceneResults2.SearchAfter.Fields, luceneResults2.SearchAfter.ShardIndex.Value); + var searchAfter2 = new SearchAfterOptions(luceneResults2.SearchAfter!.DocumentId, luceneResults2.SearchAfter.DocumentScore, luceneResults2.SearchAfter.Fields, luceneResults2.SearchAfter.ShardIndex); var luceneOptions3 = new LuceneQueryOptions(0, 1, searchAfter2); var luceneResults3 = sc.ExecuteWithLucene(luceneOptions3); Assert.IsNotNull(luceneResults3); @@ -4840,7 +5033,8 @@ public void SearchAfter_NonSorted_Results_Returns_Different_Results() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -4873,10 +5067,10 @@ public void SearchAfter_NonSorted_Results_Returns_Different_Results() Assert.IsTrue(luceneResults1List.Any(x => x.Id == "2")); // Second query result continues after result 1 (zero indexed), Takes 1, should not include any of the results before or include the SearchAfter docid / scoreid - var searchAfter = new SearchAfterOptions(luceneResults.SearchAfter.DocumentId, + var searchAfter = new SearchAfterOptions(luceneResults.SearchAfter!.DocumentId, luceneResults.SearchAfter.DocumentScore, luceneResults.SearchAfter.Fields, - luceneResults.SearchAfter.ShardIndex.Value); + luceneResults.SearchAfter.ShardIndex); var luceneOptions2 = new LuceneQueryOptions(0, 1, searchAfter); var luceneResults2 = sc.ExecuteWithLucene(luceneOptions2); var luceneResults2List = luceneResults2.ToList(); @@ -4886,7 +5080,7 @@ public void SearchAfter_NonSorted_Results_Returns_Different_Results() Assert.IsFalse(luceneResults2List.Any(x => luceneResults.ToList().Any(y => y.Id == x.Id)), "Results should not overlap"); // Third query result continues after result 2 (zero indexed), Takes 1 - var searchAfter2 = new SearchAfterOptions(luceneResults2.SearchAfter.DocumentId, luceneResults2.SearchAfter.DocumentScore, luceneResults2.SearchAfter.Fields, luceneResults2.SearchAfter.ShardIndex.Value); + var searchAfter2 = new SearchAfterOptions(luceneResults2.SearchAfter!.DocumentId, luceneResults2.SearchAfter.DocumentScore, luceneResults2.SearchAfter.Fields, luceneResults2.SearchAfter.ShardIndex); var luceneOptions3 = new LuceneQueryOptions(0, 1, searchAfter2); var luceneResults3 = sc.ExecuteWithLucene(luceneOptions3); Assert.IsNotNull(luceneResults3); @@ -4899,12 +5093,12 @@ public void SearchAfter_NonSorted_Results_Returns_Different_Results() } } -#if NET6_0_OR_GREATER - [TestCase(FacetTestType.TaxonomyFacets)] [TestCase(FacetTestType.SortedSetFacets)] + [TestCase(FacetTestType.TaxonomyFacets)] + [TestCase(FacetTestType.SortedSetFacets)] [TestCase(FacetTestType.NoFacets)] public void Range_DateOnly(FacetTestType withFacets) { - FieldDefinitionCollection fieldDefinitionCollection = null; + FieldDefinitionCollection? fieldDefinitionCollection = null; switch (withFacets) { case FacetTestType.TaxonomyFacets: @@ -4920,7 +5114,7 @@ public void Range_DateOnly(FacetTestType withFacets) var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex( + using (var indexer = GetTestIndex( luceneDir, luceneTaxonomyDir, analyzer, @@ -4964,8 +5158,9 @@ public void Range_DateOnly(FacetTestType withFacets) var numberSortedResult = numberSortedCriteria.WithFacets(facets => facets.FacetString("created")).Execute(); var facetResult = numberSortedResult.GetFacet("created"); + Assert.IsNotNull(facetResult, "Facet result should not be null"); Assert.AreEqual(2, numberSortedResult.TotalItemCount); - Assert.AreEqual(2, facetResult.Count()); + Assert.AreEqual(2, facetResult!.Count()); } else { @@ -4981,8 +5176,10 @@ public void Range_DateOnly_Min_And_Max_Inclusive() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, new FieldDefinitionCollection(new FieldDefinition("created", "datetime")))) { @@ -5030,8 +5227,10 @@ public void Range_DateOnly_No_Inclusive() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) using (var indexer = GetTestIndex( luceneDir, + luceneTaxonomyDir, analyzer, new FieldDefinitionCollection(new FieldDefinition("created", "datetime")))) { @@ -5073,7 +5272,6 @@ public void Range_DateOnly_No_Inclusive() Assert.AreEqual(1, numberSortedResult.TotalItemCount); } } -#endif [TestCase(1, 2, 1, 2)] [TestCase(2, 2, 2, 2)] @@ -5082,7 +5280,12 @@ public void GivenSearchAfterTake_Returns_ExpectedTotals_Facet(int firstTake, int const int indexSize = 5; var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer, new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)))) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex( + luceneDir, + luceneTaxonomyDir, + analyzer, + new FieldDefinitionCollection(new FieldDefinition("nodeName", FieldDefinitionTypes.FacetFullText)))) { var items = Enumerable.Range(0, indexSize).Select(x => ValueSet.FromObject(x.ToString(), "content", new { nodeName = "umbraco", headerText = "world", writerName = "administrator" })); @@ -5103,10 +5306,11 @@ public void GivenSearchAfterTake_Returns_ExpectedTotals_Facet(int firstTake, int var facetResults1 = results1.GetFacet("nodeName"); + Assert.IsNotNull(facetResults1); Assert.AreEqual(indexSize, results1.TotalItemCount); Assert.AreEqual(expectedFirstResultCount, results1.Count()); - Assert.AreEqual(1, facetResults1.Count()); - Assert.AreEqual(5, facetResults1.Facet("umbraco").Value); + Assert.AreEqual(1, facetResults1!.Count()); + Assert.AreEqual(5, facetResults1!.Facet("umbraco")!.Value); Assert.IsNotNull(results1); @@ -5114,10 +5318,11 @@ public void GivenSearchAfterTake_Returns_ExpectedTotals_Facet(int firstTake, int var facetResults2 = results2.GetFacet("nodeName"); + Assert.IsNotNull(facetResults2); Assert.AreEqual(indexSize, results2.TotalItemCount); Assert.AreEqual(expectedSecondResultCount, results2.Count()); - Assert.AreEqual(1, facetResults2.Count()); - Assert.AreEqual(5, facetResults2.Facet("umbraco").Value); + Assert.AreEqual(1, facetResults2!.Count()); + Assert.AreEqual(5, facetResults2!.Facet("umbraco")!.Value); var firstResults = results1.ToArray(); var secondResults = results2.ToArray(); Assert.IsFalse(firstResults.Any(x => secondResults.Any(y => y.Id == x.Id)), "The second set of results should not contain the first set of results"); @@ -5134,7 +5339,7 @@ public void GivenTaxonomyIndexSearchAfterTake_Returns_ExpectedTotals_Facet(int f facetConfigs.SetIndexFieldName("taxonomynodeName", "taxonomy_nodeName"); using (var luceneDir = new RandomIdRAMDirectory()) using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) - using (var indexer = GetTaxonomyTestIndex(luceneDir, luceneTaxonomyDir, analyzer, new FieldDefinitionCollection( + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer, new FieldDefinitionCollection( new FieldDefinition("nodeName", FieldDefinitionTypes.FacetTaxonomyFullText), new FieldDefinition("taxonomynodeName", FieldDefinitionTypes.FacetTaxonomyFullText) @@ -5145,18 +5350,18 @@ public void GivenTaxonomyIndexSearchAfterTake_Returns_ExpectedTotals_Facet(int f indexer.IndexItems(items); - var taxonomySearcher = indexer.TaxonomySearcher; + var taxonomySearcher = indexer.TaxonomySearcher!; var taxonomyCategoryCount = taxonomySearcher.CategoryCount; //Arrange var sc = taxonomySearcher.CreateQuery("content") .Field("writerName", "administrator") - .WithFacets((Action)(facets => + .WithFacets(facets => { facets.FacetString("nodeName"); facets.FacetString("taxonomynodeName"); - })); + }); //Act @@ -5164,26 +5369,29 @@ public void GivenTaxonomyIndexSearchAfterTake_Returns_ExpectedTotals_Facet(int f var facetResults1 = results1.GetFacet("nodeName"); + Assert.IsNotNull(facetResults1); Assert.AreEqual(indexSize, results1.TotalItemCount); Assert.AreEqual(expectedFirstResultCount, results1.Count()); - Assert.AreEqual(1, facetResults1.Count()); - Assert.AreEqual(5, facetResults1.Facet("umbraco").Value); + Assert.AreEqual(1, facetResults1!.Count()); + Assert.AreEqual(5, facetResults1!.Facet("umbraco")!.Value); Assert.IsNotNull(results1); var facetTaxonomyResults1 = results1.GetFacet("taxonomynodeName"); - Assert.AreEqual(1, facetTaxonomyResults1.Count()); - Assert.AreEqual(5, facetTaxonomyResults1.Facet("umbraco").Value); + Assert.IsNotNull(facetTaxonomyResults1); + Assert.AreEqual(1, facetTaxonomyResults1!.Count()); + Assert.AreEqual(5, facetTaxonomyResults1!.Facet("umbraco")!.Value); var results2 = sc.Execute(new LuceneQueryOptions(0, secondTake, results1.SearchAfter)); var facetResults2 = results2.GetFacet("nodeName"); var facetTaxonomyResults2 = results2.GetFacet("taxonomynodeName"); + Assert.IsNotNull(facetResults2); Assert.AreEqual(indexSize, results2.TotalItemCount); Assert.AreEqual(expectedSecondResultCount, results2.Count()); - Assert.AreEqual(1, facetResults2.Count()); - Assert.AreEqual(5, facetResults2.Facet("umbraco").Value); + Assert.AreEqual(1, facetResults2!.Count()); + Assert.AreEqual(5, facetResults2!.Facet("umbraco")!.Value); var firstResults = results1.ToArray(); var secondResults = results2.ToArray(); Assert.IsFalse(firstResults.Any(x => secondResults.Any(y => y.Id == x.Id)), "The second set of results should not contain the first set of results"); diff --git a/src/Examine.Test/Examine.Lucene/Search/LuceneSearchResultsReaderTrackerTests.cs b/src/Examine.Test/Examine.Lucene/Search/LuceneSearchResultsReaderTrackerTests.cs index a3bc90465..1b7791f44 100644 --- a/src/Examine.Test/Examine.Lucene/Search/LuceneSearchResultsReaderTrackerTests.cs +++ b/src/Examine.Test/Examine.Lucene/Search/LuceneSearchResultsReaderTrackerTests.cs @@ -16,7 +16,8 @@ public void Track_Readers() { var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir = new RandomIdRAMDirectory()) - using (var indexer = GetTestIndex(luceneDir, analyzer)) + using (var luceneTaxonomyDir = new RandomIdRAMDirectory()) + using (var indexer = GetTestIndex(luceneDir, luceneTaxonomyDir, analyzer)) { indexer.IndexItems(new[] { ValueSet.FromObject(1.ToString(), "content", @@ -32,10 +33,10 @@ public void Track_Readers() var searcher = (LuceneSearcher)indexer.Searcher; IndexReader reader; - ISearchContext searchContext = searcher.GetSearchContext(); - using (ISearcherReference searchRef = searchContext.GetSearcher()) + var searchContext = searcher.GetSearchContext(); + using (var searchRef = searchContext.GetSearcher()) { - IndexSearcher luceneSearcher = searchRef.IndexSearcher; + var luceneSearcher = searchRef.IndexSearcher; reader = luceneSearcher.IndexReader; diff --git a/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearch.cs b/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearch.cs index 514c81de3..f989dca44 100644 --- a/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearch.cs +++ b/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearch.cs @@ -4,6 +4,9 @@ using NUnit.Framework; using Examine.Lucene.Providers; using Lucene.Net.Facet; +using Moq; +using Microsoft.Extensions.Options; +using Examine.Lucene; namespace Examine.Test.Examine.Lucene.Search { @@ -17,13 +20,17 @@ public void MultiIndex_Simple_Search() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir1 = new RandomIdRAMDirectory()) using (var luceneDir2 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir2 = new RandomIdRAMDirectory()) using (var luceneDir3 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir3 = new RandomIdRAMDirectory()) using (var luceneDir4 = new RandomIdRAMDirectory()) - using (var indexer1 = GetTestIndex(luceneDir1, analyzer)) - using (var indexer2 = GetTestIndex(luceneDir2, analyzer)) - using (var indexer3 = GetTestIndex(luceneDir3, analyzer)) - using (var indexer4 = GetTestIndex(luceneDir4, analyzer)) + using (var luceneTaxonomyDir4 = new RandomIdRAMDirectory()) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir1, analyzer)) + using (var indexer2 = GetTestIndex(luceneDir2, luceneTaxonomyDir2, analyzer)) + using (var indexer3 = GetTestIndex(luceneDir3, luceneTaxonomyDir3, analyzer)) + using (var indexer4 = GetTestIndex(luceneDir4, luceneTaxonomyDir4, analyzer)) { indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value1", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." })); @@ -33,10 +40,17 @@ public void MultiIndex_Simple_Search() indexer3.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item1 = "value3", item2 = "Scotch scotch scotch, i love scotch" })); indexer4.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item1 = "value4", item2 = "60% of the time, it works everytime" })); - var searcher = new MultiIndexSearcher("testSearcher", - new[] { indexer1, indexer2, indexer3, indexer4 }, - new FacetsConfig(), - analyzer); + var luceneMultiSearcherOptions = new LuceneMultiSearcherOptions + { + IndexNames = [indexer1.Name, indexer2.Name, indexer3.Name, indexer4.Name], + Analyzer = analyzer, + FacetConfiguration = new FacetsConfig() + }; + + var searcher = new MultiIndexSearcher( + "testSearcher", + Mock.Of>(x => x.Get(It.IsAny()) == luceneMultiSearcherOptions), + new[] { indexer1, indexer2, indexer3, indexer4 }); var result = searcher.Search("darkness"); @@ -54,13 +68,17 @@ public void MultiIndex_Field_Count() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir1 = new RandomIdRAMDirectory()) using (var luceneDir2 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir2 = new RandomIdRAMDirectory()) using (var luceneDir3 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir3 = new RandomIdRAMDirectory()) using (var luceneDir4 = new RandomIdRAMDirectory()) - using (var indexer1 = GetTestIndex(luceneDir1, analyzer)) - using (var indexer2 = GetTestIndex(luceneDir2, analyzer)) - using (var indexer3 = GetTestIndex(luceneDir3, analyzer)) - using (var indexer4 = GetTestIndex(luceneDir4, analyzer)) + using (var luceneTaxonomyDir4 = new RandomIdRAMDirectory()) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir1, analyzer)) + using (var indexer2 = GetTestIndex(luceneDir2, luceneTaxonomyDir2, analyzer)) + using (var indexer3 = GetTestIndex(luceneDir3, luceneTaxonomyDir3, analyzer)) + using (var indexer4 = GetTestIndex(luceneDir4, luceneTaxonomyDir4, analyzer)) { indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "hello", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." })); indexer2.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "world", item2 = "The festival lasts five days and celebrates the victory of good over evil, light over darkness, and knowledge over ignorance." })); @@ -69,10 +87,17 @@ public void MultiIndex_Field_Count() indexer3.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item3 = "some", item2 = "Scotch scotch scotch, i love scotch" })); indexer4.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item4 = "values", item2 = "60% of the time, it works everytime" })); - var searcher = new MultiIndexSearcher("testSearcher", - new[] { indexer1, indexer2, indexer3, indexer4 }, - new FacetsConfig(), - analyzer); + var luceneMultiSearcherOptions = new LuceneMultiSearcherOptions + { + IndexNames = [indexer1.Name, indexer2.Name, indexer3.Name, indexer4.Name], + Analyzer = analyzer, + FacetConfiguration = new FacetsConfig() + }; + + var searcher = new MultiIndexSearcher( + "testSearcher", + Mock.Of>(x => x.Get(It.IsAny()) == luceneMultiSearcherOptions), + [indexer1, indexer2, indexer3, indexer4]); var result = searcher.GetSearchContext().SearchableFields; //will be item1 , item2, item3, and item4 diff --git a/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearchTests.cs b/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearchTests.cs index 4f9cf3b55..4ec66745e 100644 --- a/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearchTests.cs +++ b/src/Examine.Test/Examine.Lucene/Search/MultiIndexSearchTests.cs @@ -1,10 +1,14 @@ using System; using System.Linq; +using Examine.Lucene; using Examine.Lucene.Providers; +using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; -using NUnit.Framework; using Lucene.Net.Analysis.Util; using Lucene.Net.Facet; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; namespace Examine.Test.Examine.Lucene.Search { @@ -26,19 +30,27 @@ public void GivenCustomStopWords_WhenUsedOnlyForSearchingAndNotIndexing_TheDefau var standardAnalyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir1 = new RandomIdRAMDirectory()) using (var luceneDir2 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir2 = new RandomIdRAMDirectory()) // using the StandardAnalyzer on the indexes means that the default stop words // will get stripped from the text before being stored in the index. - using (var indexer1 = GetTestIndex(luceneDir1, standardAnalyzer)) - using (var indexer2 = GetTestIndex(luceneDir2, standardAnalyzer)) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir1, standardAnalyzer)) + using (var indexer2 = GetTestIndex(luceneDir2, luceneTaxonomyDir2, standardAnalyzer)) { indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value1", item2 = "The agitated zebras will gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." })); indexer2.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value4", item2 = "Scientists believe the lake will be home to cold-loving microbial life adapted to living in total darkness." })); + var luceneMultiSearcherOptions = new LuceneMultiSearcherOptions + { + IndexNames = [indexer1.Name, indexer2.Name], + Analyzer = customAnalyzer, + FacetConfiguration = new FacetsConfig() + }; + var searcher = new MultiIndexSearcher("testSearcher", - new[] { indexer1, indexer2 }, - new FacetsConfig(), - customAnalyzer); + Mock.Of>(x => x.Get(It.IsAny()) == luceneMultiSearcherOptions), + new[] { indexer1, indexer2 }); // Even though the custom analyzer doesn't have a stop word of 'will' // it will still return nothing because the word has been stripped during indexing. @@ -54,21 +66,29 @@ public void GivenCustomStopWords_WhenUsedOnlyForIndexingAndNotForSearching_TheDe var standardAnalyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir1 = new RandomIdRAMDirectory()) using (var luceneDir2 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir2 = new RandomIdRAMDirectory()) // using the CustomAnalyzer on the indexes means that the custom stop words // will get stripped from the text before being stored in the index. - using (var indexer1 = GetTestIndex(luceneDir1, customAnalyzer)) - using (var indexer2 = GetTestIndex(luceneDir2, customAnalyzer)) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir1, customAnalyzer)) + using (var indexer2 = GetTestIndex(luceneDir2, luceneTaxonomyDir2, customAnalyzer)) { indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value1", item2 = "The agitated zebras will gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." })); indexer2.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value4", item2 = "Scientists believe the lake will be home to cold-loving microbial life adapted to living in total darkness." })); - var searcher = new MultiIndexSearcher("testSearcher", - new[] { indexer1, indexer2 }, - new FacetsConfig(), + var luceneMultiSearcherOptions = new LuceneMultiSearcherOptions + { + IndexNames = [indexer1.Name, indexer2.Name], // The Analyzer here is used for query parsing values when // non ManagedQuery queries are executed. - standardAnalyzer); + Analyzer = standardAnalyzer, + FacetConfiguration = new FacetsConfig() + }; + + var searcher = new MultiIndexSearcher("testSearcher", + Mock.Of>(x => x.Get(It.IsAny()) == luceneMultiSearcherOptions), + new[] { indexer1, indexer2 }); // A text search like this will use a ManagedQuery which means it will // use the analyzer assigned to each field to parse the query @@ -86,21 +106,29 @@ public void GivenCustomStopWords_WhenUsedOnlyForIndexingAndNotForSearching_TheDe var standardAnalyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir1 = new RandomIdRAMDirectory()) using (var luceneDir2 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir2 = new RandomIdRAMDirectory()) // using the CustomAnalyzer on the indexes means that the custom stop words // will get stripped from the text before being stored in the index. - using (var indexer1 = GetTestIndex(luceneDir1, customAnalyzer)) - using (var indexer2 = GetTestIndex(luceneDir2, customAnalyzer)) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir1, customAnalyzer)) + using (var indexer2 = GetTestIndex(luceneDir2, luceneTaxonomyDir2, customAnalyzer)) { indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value1", item2 = "The agitated zebras will gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." })); indexer2.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value4", item2 = "Scientists believe the lake will be home to cold-loving microbial life adapted to living in total darkness." })); - var searcher = new MultiIndexSearcher("testSearcher", - new[] { indexer1, indexer2 }, - new FacetsConfig(), + var luceneMultiSearcherOptions = new LuceneMultiSearcherOptions + { + IndexNames = [indexer1.Name, indexer2.Name], // The Analyzer here is used for query parsing values when // non ManagedQuery queries are executed. - standardAnalyzer); + Analyzer = standardAnalyzer, + FacetConfiguration = new FacetsConfig() + }; + + var searcher = new MultiIndexSearcher("testSearcher", + Mock.Of>(x => x.Get(It.IsAny()) == luceneMultiSearcherOptions), + new[] { indexer1, indexer2 }); var result = searcher .CreateQuery("content") @@ -119,13 +147,17 @@ public void MultiIndex_Simple_Search() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir1 = new RandomIdRAMDirectory()) using (var luceneDir2 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir2 = new RandomIdRAMDirectory()) using (var luceneDir3 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir3 = new RandomIdRAMDirectory()) using (var luceneDir4 = new RandomIdRAMDirectory()) - using (var indexer1 = GetTestIndex(luceneDir1, analyzer)) - using (var indexer2 = GetTestIndex(luceneDir2, analyzer)) - using (var indexer3 = GetTestIndex(luceneDir3, analyzer)) - using (var indexer4 = GetTestIndex(luceneDir4, analyzer)) + using (var luceneTaxonomyDir4 = new RandomIdRAMDirectory()) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir1, analyzer)) + using (var indexer2 = GetTestIndex(luceneDir2, luceneTaxonomyDir2, analyzer)) + using (var indexer3 = GetTestIndex(luceneDir3, luceneTaxonomyDir3, analyzer)) + using (var indexer4 = GetTestIndex(luceneDir4, luceneTaxonomyDir4, analyzer)) { indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value1", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." })); @@ -135,10 +167,16 @@ public void MultiIndex_Simple_Search() indexer3.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item1 = "value3", item2 = "Scotch scotch scotch, i love scotch" })); indexer4.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item1 = "value4", item2 = "60% of the time, it works everytime" })); + var luceneMultiSearcherOptions = new LuceneMultiSearcherOptions + { + IndexNames = [indexer1.Name, indexer2.Name, indexer3.Name, indexer4.Name], + Analyzer = analyzer, + FacetConfiguration = new FacetsConfig() + }; + var searcher = new MultiIndexSearcher("testSearcher", - new[] { indexer1, indexer2, indexer3, indexer4 }, - new FacetsConfig(), - analyzer); + Mock.Of>(x => x.Get(It.IsAny()) == luceneMultiSearcherOptions), + new[] { indexer1, indexer2, indexer3, indexer4 }); var result = searcher.Search("darkness"); @@ -156,13 +194,17 @@ public void MultiIndex_Field_Count() var analyzer = new StandardAnalyzer(LuceneInfo.CurrentVersion); using (var luceneDir1 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir1 = new RandomIdRAMDirectory()) using (var luceneDir2 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir2 = new RandomIdRAMDirectory()) using (var luceneDir3 = new RandomIdRAMDirectory()) + using (var luceneTaxonomyDir3 = new RandomIdRAMDirectory()) using (var luceneDir4 = new RandomIdRAMDirectory()) - using (var indexer1 = GetTestIndex(luceneDir1, analyzer)) - using (var indexer2 = GetTestIndex(luceneDir2, analyzer)) - using (var indexer3 = GetTestIndex(luceneDir3, analyzer)) - using (var indexer4 = GetTestIndex(luceneDir4, analyzer)) + using (var luceneTaxonomyDir4 = new RandomIdRAMDirectory()) + using (var indexer1 = GetTestIndex(luceneDir1, luceneTaxonomyDir1, analyzer)) + using (var indexer2 = GetTestIndex(luceneDir2, luceneTaxonomyDir2, analyzer)) + using (var indexer3 = GetTestIndex(luceneDir3, luceneTaxonomyDir3, analyzer)) + using (var indexer4 = GetTestIndex(luceneDir4, luceneTaxonomyDir4, analyzer)) { indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "hello", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." })); indexer2.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "world", item2 = "The festival lasts five days and celebrates the victory of good over evil, light over darkness, and knowledge over ignorance." })); @@ -171,11 +213,17 @@ public void MultiIndex_Field_Count() indexer3.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item3 = "some", item2 = "Scotch scotch scotch, i love scotch" })); indexer4.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item4 = "values", item2 = "60% of the time, it works everytime" })); + var luceneMultiSearcherOptions = new LuceneMultiSearcherOptions + { + IndexNames = [indexer1.Name, indexer2.Name, indexer3.Name, indexer4.Name], + Analyzer = analyzer, + FacetConfiguration = new FacetsConfig() + }; + var searcher = new MultiIndexSearcher( "testSearcher", - new[] { indexer1, indexer2, indexer3, indexer4 }, - new FacetsConfig(), - analyzer); + Mock.Of>(x => x.Get(It.IsAny()) == luceneMultiSearcherOptions), + new[] { indexer1, indexer2, indexer3, indexer4 }); var searchContext = searcher.GetSearchContext(); var result = searchContext.SearchableFields; diff --git a/src/Examine.Test/Examine.Test.csproj b/src/Examine.Test/Examine.Test.csproj index 9a5e64bc4..7f566dbf3 100644 --- a/src/Examine.Test/Examine.Test.csproj +++ b/src/Examine.Test/Examine.Test.csproj @@ -1,17 +1,9 @@ Library - - - - - - - - - net7.0;net6.0; + net6.0;net8.0; false false @@ -53,21 +45,18 @@ - - - 4.8.0-beta00016 - - - - - - - - 3.13.3 - - + + + + + + + + + - + + diff --git a/src/Examine.Test/ExamineBaseTest.cs b/src/Examine.Test/ExamineBaseTest.cs index a75b67417..3e8030b22 100644 --- a/src/Examine.Test/ExamineBaseTest.cs +++ b/src/Examine.Test/ExamineBaseTest.cs @@ -1,67 +1,73 @@ -using NUnit.Framework; +using System.Collections.Generic; +using Examine.Lucene; +using Examine.Lucene.Directories; +using Lucene.Net.Analysis; +using Lucene.Net.Facet; +using Lucene.Net.Facet.Taxonomy.Directory; using Lucene.Net.Index; +using Lucene.Net.Replicator; using Microsoft.Extensions.Logging; -using Lucene.Net.Analysis; -using Directory = Lucene.Net.Store.Directory; using Microsoft.Extensions.Options; -using Examine.Lucene; using Moq; -using Examine.Lucene.Directories; -using System.Collections.Generic; -using Lucene.Net.Facet; +using NUnit.Framework; +using Directory = Lucene.Net.Store.Directory; namespace Examine.Test { public abstract class ExamineBaseTest { + protected ILoggerFactory LoggerFactory => CreateLoggerFactory(); + [SetUp] public virtual void Setup() { - var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); - loggerFactory.CreateLogger(typeof(ExamineBaseTest)).LogDebug("Initializing test"); } - public TestIndex GetTestIndex(Directory d, Analyzer analyzer, FieldDefinitionCollection fieldDefinitions = null, IndexDeletionPolicy indexDeletionPolicy = null, IReadOnlyDictionary indexValueTypesFactory = null, FacetsConfig facetsConfig = null) - { - var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); - return new TestIndex( - loggerFactory, + [TearDown] + public virtual void TearDown() => LoggerFactory.Dispose(); + + public TestIndex GetTestIndex( + Directory luceneDir, + Directory taxonomyDir, + Analyzer analyzer, + FieldDefinitionCollection? fieldDefinitions = null, + IndexDeletionPolicy? indexDeletionPolicy = null, + IReadOnlyDictionary? indexValueTypesFactory = null, + double nrtTargetMaxStaleSec = 60, + double nrtTargetMinStaleSec = 1, + bool nrtEnabled = true, + FacetsConfig? facetsConfig = null) + => new TestIndex( + LoggerFactory, Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions { - FieldDefinitions = fieldDefinitions, - DirectoryFactory = new GenericDirectoryFactory(_ => d, null), + FieldDefinitions = fieldDefinitions ?? new FieldDefinitionCollection(), + DirectoryFactory = GenericDirectoryFactory.FromExternallyManaged(_ => luceneDir, _ => taxonomyDir), Analyzer = analyzer, IndexDeletionPolicy = indexDeletionPolicy, IndexValueTypesFactory = indexValueTypesFactory, + NrtTargetMaxStaleSec = nrtTargetMaxStaleSec, + NrtTargetMinStaleSec = nrtTargetMinStaleSec, + NrtEnabled = nrtEnabled, FacetsConfig = facetsConfig ?? new FacetsConfig() })); - } - - public TestIndex GetTestIndex(IndexWriter writer) - { - var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); - return new TestIndex( - loggerFactory, - Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneIndexOptions()), - writer); - } - public TestIndex GetTaxonomyTestIndex(Directory d, Directory taxonomyDirectory, Analyzer analyzer, FieldDefinitionCollection fieldDefinitions = null, IndexDeletionPolicy indexDeletionPolicy = null, IReadOnlyDictionary indexValueTypesFactory = null, FacetsConfig facetsConfig = null) - { - var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); - return new TestIndex( - loggerFactory, - Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + public TestIndex GetTestIndex( + IndexWriter writer, + SnapshotDirectoryTaxonomyIndexWriterFactory taxonomyWriterFactory, + double nrtTargetMaxStaleSec = 60, + double nrtTargetMinStaleSec = 1) + => new TestIndex( + LoggerFactory, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneIndexOptions { - FieldDefinitions = fieldDefinitions, - DirectoryFactory = new GenericDirectoryFactory(_ => d, _ => taxonomyDirectory), - Analyzer = analyzer, - IndexDeletionPolicy = indexDeletionPolicy, - IndexValueTypesFactory = indexValueTypesFactory, - FacetsConfig = facetsConfig ?? new FacetsConfig(), - UseTaxonomyIndex = true - })); - } + NrtTargetMaxStaleSec = nrtTargetMaxStaleSec, + NrtTargetMinStaleSec = nrtTargetMinStaleSec + }), + writer, + taxonomyWriterFactory); + protected virtual ILoggerFactory CreateLoggerFactory() + => Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); } } diff --git a/src/Examine.Test/ExamineExtensions.cs b/src/Examine.Test/ExamineExtensions.cs index 6a9b4db7a..9583a8816 100644 --- a/src/Examine.Test/ExamineExtensions.cs +++ b/src/Examine.Test/ExamineExtensions.cs @@ -18,7 +18,7 @@ public static class ExamineExtensions /// internal static bool IsExamineElement(this XElement x) { - var id = (string) x.Attribute("id"); + var id = (string?)x.Attribute("id"); if (string.IsNullOrEmpty(id)) { return false; @@ -39,11 +39,11 @@ internal static bool IsExamineElement(this XElement x) /// /// /// - internal static string ExamineNodeTypeAlias(this XElement x) + internal static string? ExamineNodeTypeAlias(this XElement x) { - return string.IsNullOrEmpty(((string) x.Attribute("nodeTypeAlias"))) + return string.IsNullOrEmpty((string?)x.Attribute("nodeTypeAlias")) ? x.Name.LocalName - : (string) x.Attribute("nodeTypeAlias"); + : (string?)x.Attribute("nodeTypeAlias"); } /// @@ -54,12 +54,12 @@ internal static string ExamineNodeTypeAlias(this XElement x) /// internal static string SelectExamineDataValue(this XElement xml, string alias) { - XElement nodeData = null; + XElement? nodeData = null; //if there is data children with attributes, we're on the old if (xml.Elements("data").Any(x => x.HasAttributes)) { - nodeData = xml.Elements("data").SingleOrDefault(x => string.Equals(((string) x.Attribute("alias")), alias, StringComparison.InvariantCultureIgnoreCase)); + nodeData = xml.Elements("data").SingleOrDefault(x => string.Equals((string?)x.Attribute("alias"), alias, StringComparison.InvariantCultureIgnoreCase)); } else { @@ -90,14 +90,14 @@ public static ValueSet ConvertToValueSet(this XElement xml, string indexCategory throw new InvalidOperationException("Not a supported Examine XML structure"); } - var id = (string)xml.Attribute("id"); + var id = (string?)xml.Attribute("id"); //we will use this as the item type, but we also need to add this as the 'nodeTypeAlias' as part of the properties //since this is what Umbraco expects var nodeTypeAlias = xml.ExamineNodeTypeAlias(); var allVals = xml.SelectExamineAllValues(); - allVals["nodeTypeAlias"] = nodeTypeAlias; + allVals["nodeTypeAlias"] = nodeTypeAlias!; - var set = new ValueSet(id, indexCategory, nodeTypeAlias, allVals); + var set = new ValueSet(id!, indexCategory, nodeTypeAlias!, allVals); return set; } @@ -123,13 +123,13 @@ internal static Dictionary SelectExamineDataValues(this XElement if (x.Attribute("id") != null) { continue; - } + } string key; if (x.Name.LocalName == "data") { //it's the legacy schema - key = (string)x.Attribute("alias"); + key = (string)x.Attribute("alias")!; } else { diff --git a/src/Examine.Test/IndexInitializer.cs b/src/Examine.Test/IndexInitializer.cs index ec73539d2..9ce11648d 100644 --- a/src/Examine.Test/IndexInitializer.cs +++ b/src/Examine.Test/IndexInitializer.cs @@ -123,7 +123,7 @@ internal static class IndexInitializer //} - internal static void IndexingError(object sender, IndexingErrorEventArgs e) => throw new ApplicationException(e.Message, e.Exception); + internal static void IndexingError(object? sender, IndexingErrorEventArgs e) => throw new ApplicationException(e.Message, e.Exception); } diff --git a/src/Examine.Test/OrderedDictionaryTests.cs b/src/Examine.Test/OrderedDictionaryTests.cs index ce92e28b3..141225b5f 100644 --- a/src/Examine.Test/OrderedDictionaryTests.cs +++ b/src/Examine.Test/OrderedDictionaryTests.cs @@ -11,7 +11,7 @@ namespace Examine.Test [TestFixture] public class OrderedDictionaryTests { - private OrderedDictionary GetAlphabetDictionary(IEqualityComparer comparer = null) + private OrderedDictionary GetAlphabetDictionary(IEqualityComparer? comparer = null) { var alphabet = (comparer == null ? new OrderedDictionary() : new OrderedDictionary(comparer)); for (var a = Convert.ToInt32('a'); a <= Convert.ToInt32('z'); a++) @@ -115,7 +115,7 @@ public void TestTryGetValue() { var alphabetDict = GetAlphabetDictionary(); #pragma warning disable IDE0018 // Inline variable declaration - string result = null; + string? result = null; #pragma warning restore IDE0018 // Inline variable declaration Assert.IsFalse(alphabetDict.TryGetValue("abc", out result)); Assert.IsNull(result); diff --git a/src/Examine.Test/TestContentService.cs b/src/Examine.Test/TestContentService.cs index fa00a5014..7733f23c8 100644 --- a/src/Examine.Test/TestContentService.cs +++ b/src/Examine.Test/TestContentService.cs @@ -12,7 +12,7 @@ namespace Examine.Test /// public class TestContentService { - private XDocument _xDoc; + private XDocument? _xDoc; /// /// Return the XDocument containing the xml from the umbraco.config xml file @@ -32,7 +32,7 @@ public XDocument GetPublishedContentByXPath(string xpath) } var xdoc = XDocument.Parse(""); - xdoc.Root.Add(_xDoc.XPathSelectElements(xpath)); + xdoc.Root!.Add(_xDoc.XPathSelectElements(xpath)); return xdoc; } diff --git a/src/Examine.Test/TestIndex.cs b/src/Examine.Test/TestIndex.cs index a6dd1841e..7763764c3 100644 --- a/src/Examine.Test/TestIndex.cs +++ b/src/Examine.Test/TestIndex.cs @@ -1,7 +1,9 @@ using System.Collections.Generic; using Examine.Lucene; using Examine.Lucene.Providers; +using Lucene.Net.Facet.Taxonomy.Directory; using Lucene.Net.Index; +using Lucene.Net.Replicator; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -17,13 +19,13 @@ public TestIndex(ILoggerFactory loggerFactory, IOptionsMonitor options, IndexWriter writer) - : base(loggerFactory, TestIndexName, options, writer) + public TestIndex(ILoggerFactory loggerFactory, IOptionsMonitor options, IndexWriter writer, SnapshotDirectoryTaxonomyIndexWriterFactory taxonomyWriterFactory) + : base(loggerFactory, TestIndexName, options, writer, taxonomyWriterFactory) { RunAsync = false; } - public IEnumerable AllData() + public static IEnumerable AllData() { var data = new List(); for (int i = 0; i < 100; i++) diff --git a/src/Examine.Web.Demo/ConfigureIndexOptions.cs b/src/Examine.Web.Demo/ConfigureIndexOptions.cs index 3466d8f72..7b476e623 100644 --- a/src/Examine.Web.Demo/ConfigureIndexOptions.cs +++ b/src/Examine.Web.Demo/ConfigureIndexOptions.cs @@ -17,7 +17,7 @@ public ConfigureIndexOptions(ILoggerFactory loggerFactory) _loggerFactory = loggerFactory; } - public void Configure(string name, LuceneDirectoryIndexOptions options) + public void Configure(string? name, LuceneDirectoryIndexOptions options) { switch (name) { @@ -41,7 +41,6 @@ public void Configure(string name, LuceneDirectoryIndexOptions options) options.FieldDefinitions.AddOrUpdate(new FieldDefinition("phone", "phone")); break; case "TaxonomyFacetIndex": - options.UseTaxonomyIndex = true; options.FacetsConfig.SetMultiValued("Tags", true); options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressState", FieldDefinitionTypes.FacetTaxonomyFullText)); options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressStateCity", FieldDefinitionTypes.FacetTaxonomyFullText)); @@ -49,7 +48,6 @@ public void Configure(string name, LuceneDirectoryIndexOptions options) break; case "FacetIndex": - options.UseTaxonomyIndex = false; options.FacetsConfig.SetMultiValued("Tags", true); options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressState", FieldDefinitionTypes.FacetFullText)); options.FieldDefinitions.AddOrUpdate(new FieldDefinition("AddressStateCity", FieldDefinitionTypes.FacetFullText)); diff --git a/src/Examine.Web.Demo/Examine.Web.Demo.csproj b/src/Examine.Web.Demo/Examine.Web.Demo.csproj index 1b9b52298..18d4201c5 100644 --- a/src/Examine.Web.Demo/Examine.Web.Demo.csproj +++ b/src/Examine.Web.Demo/Examine.Web.Demo.csproj @@ -7,6 +7,7 @@ + @@ -14,10 +15,6 @@ - - - - diff --git a/src/Examine.Web.Demo/IndexFactoryExtensions.cs b/src/Examine.Web.Demo/IndexFactoryExtensions.cs index 91729169a..49cc16a17 100644 --- a/src/Examine.Web.Demo/IndexFactoryExtensions.cs +++ b/src/Examine.Web.Demo/IndexFactoryExtensions.cs @@ -1,4 +1,6 @@ using Lucene.Net.Facet; +using Examine.Lucene.Directories; +using Examine.Lucene.Providers; using Microsoft.Extensions.DependencyInjection; namespace Examine.Web.Demo @@ -10,9 +12,9 @@ public static class IndexFactoryExtensions { public static IServiceCollection CreateIndexes(this IServiceCollection services) { - services.AddExamineLuceneIndex("MyIndex"); + services.AddExamineLuceneIndex("MyIndex"); - services.AddExamineLuceneIndex("SyncedIndex"); + services.AddExamineLuceneIndex("SyncedIndex"); var taxonomyFacetIndexFacetsConfig = new FacetsConfig(); taxonomyFacetIndexFacetsConfig.SetIndexFieldName("AddressState", "AddressState"); @@ -38,7 +40,7 @@ public static IServiceCollection CreateIndexes(this IServiceCollection services) services.AddExamineLuceneMultiSearcher( "MultiIndexSearcher", new[] { "MyIndex", "SyncedIndex", "FacetIndex" }, - opt=> opt.FacetConfiguration = new FacetsConfig()); + opt => opt.FacetConfiguration = new FacetsConfig()); services.ConfigureOptions(); diff --git a/src/Examine.sln b/src/Examine.sln index 4841ded41..5075c0044 100644 --- a/src/Examine.sln +++ b/src/Examine.sln @@ -15,8 +15,8 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{EE632768-6ECD-40EF-84BD-5594670EA7A8}" ProjectSection(SolutionItems) = preProject ..\.github\workflows\build.yml = ..\.github\workflows\build.yml - ..\GitVersion.yml = ..\GitVersion.yml ..\.github\workflows\test-report.yml = ..\.github\workflows\test-report.yml + ..\GitVersion.yml.bak = ..\GitVersion.yml.bak EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Examine.Lucene", "Examine.Lucene\Examine.Lucene.csproj", "{DA5E35C3-89BA-4A2E-A559-32CF7B23CBFF}" @@ -29,6 +29,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Examine", "Examine.Host\Exa EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Examine.Web.Demo", "Examine.Web.Demo\Examine.Web.Demo.csproj", "{99D0B284-AFDA-4A32-A88B-9B182DF8CE2F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Examine.Benchmarks", "Examine.Benchmarks\Examine.Benchmarks.csproj", "{07D99A13-2B8B-4D13-90FE-0AB1F555C92D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -55,6 +57,10 @@ Global {99D0B284-AFDA-4A32-A88B-9B182DF8CE2F}.Debug|Any CPU.Build.0 = Debug|Any CPU {99D0B284-AFDA-4A32-A88B-9B182DF8CE2F}.Release|Any CPU.ActiveCfg = Release|Any CPU {99D0B284-AFDA-4A32-A88B-9B182DF8CE2F}.Release|Any CPU.Build.0 = Release|Any CPU + {07D99A13-2B8B-4D13-90FE-0AB1F555C92D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {07D99A13-2B8B-4D13-90FE-0AB1F555C92D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {07D99A13-2B8B-4D13-90FE-0AB1F555C92D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {07D99A13-2B8B-4D13-90FE-0AB1F555C92D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE