benchmarks

searchpioneer · May 23, 2024 · 188c722 · 188c722
1 parent 6ed3797
commit 188c722
Show file tree

Hide file tree

Showing 12 changed files with 407 additions and 24 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,28 @@
+name: Benchmark
+
+on:
+  workflow_dispatch:
+    inputs: 
+      tests:
+        type: string
+        required: true
+        default: "*English*Detection*"
+        description: "filter for tests to run"
+
+jobs:
+  report:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.0.x
+      - name: Run Benchmarks
+        run: ./build.sh benchmark --filter ${{ inputs.tests }} --exporters GitHub --job Short
+      - name: Write Summary
+        run: |
+          cat BenchmarkDotNet.Artifacts/results/*github.md >> $GITHUB_STEP_SUMMARY
+          
diff --git a/src/Lingua/LanguageDetector.cs b/src/Lingua/LanguageDetector.cs
@@ -445,8 +445,9 @@ private void PreloadLanguageModels()
 			using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
 			return LanguageModel.FromJson(gzipStream);
 		}
-		catch (Exception ex) when (ex is FileNotFoundException or IOException)
+		catch (FileNotFoundException)
 		{
+			// there may not be a model for a given ngram/language
 			return new Dictionary<string, double>();
 		}
 	}

diff --git a/src/Lingua/Lingua.csproj b/src/Lingua/Lingua.csproj
@@ -76,7 +76,7 @@
     </GZip>
     <ItemGroup>
       <Content Remove="@(Content)" />
-      <Content Include="@(GZipContent)" CopyToOutputDirectory="PreserveNewest">
+      <Content Include="@(GZipContent)" CopyToOutputDirectory="Always">
         <!-- Package gzipped contents -->
         <Pack>true</Pack>
         <PackagePath>contentFiles/any/any/%(RelativeDir)%(Filename)%(Extension)</PackagePath>

diff --git a/tests/Lingua.AccuracyReport.Tests/SupportedLanguages.cs b/tests/Lingua.AccuracyReport.Tests/SupportedLanguages.cs
@@ -19,10 +19,6 @@ static SupportedLanguages()
 				languagesSupportedByLanguageDetection.Add(Nynorsk);
 				languagesSupportedByLanguageDetection.Add(Bokmal);
 			}
-			else
-			{
-				var foo = isoCode;
-			}
 		}
 
 		var languageDetection = languagesSupportedByLanguageDetection.ToArray();

diff --git a/tests/Lingua.Benchmarks/Lingua.Benchmarks.csproj b/tests/Lingua.Benchmarks/Lingua.Benchmarks.csproj
@@ -4,7 +4,7 @@
         <OutputType>Exe</OutputType>
         <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
+        <Nullable>disable</Nullable>
     </PropertyGroup>
 
   <ItemGroup>

diff --git a/tests/Lingua.Benchmarks/Sentences/EnglishSentencesDetection.cs b/tests/Lingua.Benchmarks/Sentences/EnglishSentencesDetection.cs
@@ -0,0 +1,59 @@
+using BenchmarkDotNet.Attributes;
+using NTextCat;
+
+namespace Lingua.Benchmarks.Sentences;
+
+public class EnglishSentencesDetection
+{
+	private LanguageDetector _linguaLanguageDetector;
+	private LanguageDetector _lowAccuracyLinguaLanguageDetector;
+	private LanguageDetection.LanguageDetector _languageDetectionLanguageDetector;
+	private RankedLanguageIdentifier _nTextCatLanguageDetector;
+
+	[GlobalSetup]
+	public void GlobalSetup()
+	{
+		var languages = SupportedLanguages.ByAllImplementations;
+
+		var detector = new LanguageDetection.LanguageDetector();
+		detector.AddLanguages(languages.Select(l => l.IsoCode6393().ToString().ToLowerInvariant()).ToArray());
+		_languageDetectionLanguageDetector = detector;
+
+		_linguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.Build();
+
+		_lowAccuracyLinguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.WithLowAccuracyMode()
+			.Build();
+
+		var factory = new RankedLanguageIdentifierFactory();
+		using var stream = typeof(Program).Assembly
+			.GetManifestResourceStream("Lingua.Benchmarks.Core14.profile.xml");
+		var nTextCatDetector = factory.Load(stream);
+		_nTextCatLanguageDetector = nTextCatDetector;
+	}
+
+	[Benchmark(Baseline = true)]
+	public Language LinguaLowAccuracy() => _lowAccuracyLinguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public Language Lingua() => _linguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public string LanguageDetection() => _languageDetectionLanguageDetector.Detect(Text);
+
+	[Benchmark]
+	public Tuple<NTextCat.LanguageInfo, double> NTextCat() => _nTextCatLanguageDetector.Identify(Text).First();
+
+	[ParamsSource(nameof(ValuesForText))]
+	public string Text { get; set; }
+
+	public IEnumerable<string> ValuesForText => new[]
+	{
+		"On numerous occasions, slight disturbances have caused computers to malfunction and entire industrial facilities to shutdown."
+	};
+}
diff --git a/.../Lingua.Benchmarks/SingleWordDetection.cs → .../SingleWord/EnglishSingleWordDetection.cs b/.../Lingua.Benchmarks/SingleWordDetection.cs → .../SingleWord/EnglishSingleWordDetection.cs
@@ -1,17 +1,17 @@
-using BenchmarkDotNet.Attributes;
-using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Attributes;
 using NTextCat;
 
-namespace Lingua.Benchmarks;
+namespace Lingua.Benchmarks.SingleWord;
 
-public class SingleWordDetection
+public class EnglishSingleWordDetection
 {
-	private readonly LanguageDetector _linguaLanguageDetector;
-	private readonly LanguageDetector _lowAccuracyLinguaLanguageDetector;
-	private readonly LanguageDetection.LanguageDetector _languageDetectionLanguageDetector;
-	private readonly RankedLanguageIdentifier _nTextCatLanguageDetector;
+	private LanguageDetector _linguaLanguageDetector;
+	private LanguageDetector _lowAccuracyLinguaLanguageDetector;
+	private LanguageDetection.LanguageDetector _languageDetectionLanguageDetector;
+	private RankedLanguageIdentifier _nTextCatLanguageDetector;
 
-	public SingleWordDetection()
+	[GlobalSetup]
+	public void GlobalSetup()
 	{
 		var languages = SupportedLanguages.ByAllImplementations;
 
@@ -37,19 +37,23 @@ public SingleWordDetection()
 		_nTextCatLanguageDetector = nTextCatDetector;
 	}
 
-	[Params("ialomiţa", "podĺa", "ґрунтовому", "cằm", "suspiciously")]
-	// ReSharper disable once UnassignedField.Global
-	public string? Text;
-
 	[Benchmark(Baseline = true)]
-	public Language LinguaLowAccuracy() => _lowAccuracyLinguaLanguageDetector.DetectLanguageOf(Text!);
+	public Language LinguaLowAccuracy() => _lowAccuracyLinguaLanguageDetector.DetectLanguageOf(Text);
 
 	[Benchmark]
-	public Language Lingua() => _linguaLanguageDetector.DetectLanguageOf(Text!);
+	public Language Lingua() => _linguaLanguageDetector.DetectLanguageOf(Text);
 
 	[Benchmark]
-	public string? LanguageDetection() => _languageDetectionLanguageDetector.Detect(Text!);
+	public string LanguageDetection() => _languageDetectionLanguageDetector.Detect(Text);
 
 	[Benchmark]
-	public Tuple<NTextCat.LanguageInfo, double> NTextCat() => _nTextCatLanguageDetector.Identify(Text!).First();
+	public Tuple<NTextCat.LanguageInfo, double> NTextCat() => _nTextCatLanguageDetector.Identify(Text).First();
+
+	[ParamsSource(nameof(ValuesForText))]
+	public string Text { get; set; }
+
+	public IEnumerable<string> ValuesForText => new[]
+	{
+		"suspiciously"
+	};
 }
diff --git a/tests/Lingua.Benchmarks/SingleWord/RomanianSingleWordDetection.cs b/tests/Lingua.Benchmarks/SingleWord/RomanianSingleWordDetection.cs
@@ -0,0 +1,59 @@
+using BenchmarkDotNet.Attributes;
+using NTextCat;
+
+namespace Lingua.Benchmarks.SingleWord;
+
+public class RomanianSingleWordDetection
+{
+	private LanguageDetector _linguaLanguageDetector;
+	private LanguageDetector _lowAccuracyLinguaLanguageDetector;
+	private LanguageDetection.LanguageDetector _languageDetectionLanguageDetector;
+	private RankedLanguageIdentifier _nTextCatLanguageDetector;
+
+	[GlobalSetup]
+	public void GlobalSetup()
+	{
+		var languages = SupportedLanguages.ByAllImplementations;
+
+		var detector = new LanguageDetection.LanguageDetector();
+		detector.AddLanguages(languages.Select(l => l.IsoCode6393().ToString().ToLowerInvariant()).ToArray());
+		_languageDetectionLanguageDetector = detector;
+
+		_linguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.Build();
+
+		_lowAccuracyLinguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.WithLowAccuracyMode()
+			.Build();
+
+		var factory = new RankedLanguageIdentifierFactory();
+		using var stream = typeof(Program).Assembly
+			.GetManifestResourceStream("Lingua.Benchmarks.Core14.profile.xml");
+		var nTextCatDetector = factory.Load(stream);
+		_nTextCatLanguageDetector = nTextCatDetector;
+	}
+
+	[Benchmark(Baseline = true)]
+	public Language LinguaLowAccuracy() => _lowAccuracyLinguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public Language Lingua() => _linguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public string LanguageDetection() => _languageDetectionLanguageDetector.Detect(Text);
+
+	[Benchmark]
+	public Tuple<NTextCat.LanguageInfo, double> NTextCat() => _nTextCatLanguageDetector.Identify(Text).First();
+
+	[ParamsSource(nameof(ValuesForText))]
+	public string Text { get; set; }
+
+	public IEnumerable<string> ValuesForText => new[]
+	{
+		"ialomiţa"
+	};
+}
diff --git a/tests/Lingua.Benchmarks/SingleWord/SlovakSingleWordDetection.cs b/tests/Lingua.Benchmarks/SingleWord/SlovakSingleWordDetection.cs
@@ -0,0 +1,59 @@
+using BenchmarkDotNet.Attributes;
+using NTextCat;
+
+namespace Lingua.Benchmarks.SingleWord;
+
+public class SlovakSingleWordDetection
+{
+	private LanguageDetector _linguaLanguageDetector;
+	private LanguageDetector _lowAccuracyLinguaLanguageDetector;
+	private LanguageDetection.LanguageDetector _languageDetectionLanguageDetector;
+	private RankedLanguageIdentifier _nTextCatLanguageDetector;
+
+	[GlobalSetup]
+	public void GlobalSetup()
+	{
+		var languages = SupportedLanguages.ByAllImplementations;
+
+		var detector = new LanguageDetection.LanguageDetector();
+		detector.AddLanguages(languages.Select(l => l.IsoCode6393().ToString().ToLowerInvariant()).ToArray());
+		_languageDetectionLanguageDetector = detector;
+
+		_linguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.Build();
+
+		_lowAccuracyLinguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.WithLowAccuracyMode()
+			.Build();
+
+		var factory = new RankedLanguageIdentifierFactory();
+		using var stream = typeof(Program).Assembly
+			.GetManifestResourceStream("Lingua.Benchmarks.Core14.profile.xml");
+		var nTextCatDetector = factory.Load(stream);
+		_nTextCatLanguageDetector = nTextCatDetector;
+	}
+
+	[Benchmark(Baseline = true)]
+	public Language LinguaLowAccuracy() => _lowAccuracyLinguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public Language Lingua() => _linguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public string LanguageDetection() => _languageDetectionLanguageDetector.Detect(Text);
+
+	[Benchmark]
+	public Tuple<NTextCat.LanguageInfo, double> NTextCat() => _nTextCatLanguageDetector.Identify(Text).First();
+
+	[ParamsSource(nameof(ValuesForText))]
+	public string Text { get; set; }
+
+	public IEnumerable<string> ValuesForText => new[]
+	{
+		"podĺa"
+	};
+}
diff --git a/tests/Lingua.Benchmarks/SingleWord/UkranianSingleWordDetection.cs b/tests/Lingua.Benchmarks/SingleWord/UkranianSingleWordDetection.cs
@@ -0,0 +1,59 @@
+using BenchmarkDotNet.Attributes;
+using NTextCat;
+
+namespace Lingua.Benchmarks.SingleWord;
+
+public class UkranianSingleWordDetection
+{
+	private LanguageDetector _linguaLanguageDetector;
+	private LanguageDetector _lowAccuracyLinguaLanguageDetector;
+	private LanguageDetection.LanguageDetector _languageDetectionLanguageDetector;
+	private RankedLanguageIdentifier _nTextCatLanguageDetector;
+
+	[GlobalSetup]
+	public void GlobalSetup()
+	{
+		var languages = SupportedLanguages.ByAllImplementations;
+
+		var detector = new LanguageDetection.LanguageDetector();
+		detector.AddLanguages(languages.Select(l => l.IsoCode6393().ToString().ToLowerInvariant()).ToArray());
+		_languageDetectionLanguageDetector = detector;
+
+		_linguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.Build();
+
+		_lowAccuracyLinguaLanguageDetector = LanguageDetectorBuilder
+			.FromLanguages(languages)
+			.WithPreloadedLanguageModels()
+			.WithLowAccuracyMode()
+			.Build();
+
+		var factory = new RankedLanguageIdentifierFactory();
+		using var stream = typeof(Program).Assembly
+			.GetManifestResourceStream("Lingua.Benchmarks.Core14.profile.xml");
+		var nTextCatDetector = factory.Load(stream);
+		_nTextCatLanguageDetector = nTextCatDetector;
+	}
+
+	[Benchmark(Baseline = true)]
+	public Language LinguaLowAccuracy() => _lowAccuracyLinguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public Language Lingua() => _linguaLanguageDetector.DetectLanguageOf(Text);
+
+	[Benchmark]
+	public string LanguageDetection() => _languageDetectionLanguageDetector.Detect(Text);
+
+	[Benchmark]
+	public Tuple<NTextCat.LanguageInfo, double> NTextCat() => _nTextCatLanguageDetector.Identify(Text).First();
+
+	[ParamsSource(nameof(ValuesForText))]
+	public string Text { get; set; }
+
+	public IEnumerable<string> ValuesForText => new[]
+	{
+		"ґрунтовому"
+	};
+}