Skip to content

Commit 8d4b269

Browse files
svengeancejnyrup
svengeance
and
jnyrup
authored
Benchmarks Introduction (#135)
* #127 - Initial suite of benchmarks * Finishes two "real-world" benchmark scenarios * Adds readme * Simplifies logic for delayed execution * Properly initializes LazyCache with all options not using default init - updates benchmarks * Remove unnecessary async/await from immediately-returned Tasks Co-authored-by: Jonas Nyrup <[email protected]> * Remove unnecessary async/await from immediately-returned Tasks Co-authored-by: Jonas Nyrup <[email protected]> * Adds additional benchmarks for cache hits/misses & updates readme Co-authored-by: Jonas Nyrup <[email protected]>
1 parent e38695b commit 8d4b269

10 files changed

+375
-3
lines changed

‎.gitignore

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,16 @@ Desktop.ini
187187

188188
# Recycle Bin used on file shares
189189
$RECYCLE.BIN/
190-
191-
packages/
192-
.vs/
190+
191+
packages/
192+
.vs/
193+
194+
# Benchmark Dot Net
195+
**/BenchmarkDotNet.Artifacts/*
196+
**/project.lock.json
197+
tests/output/*
198+
.vs/restore.dg
199+
artifacts/*
200+
BDN.Generated
201+
BenchmarkDotNet.Samples/Properties/launchSettings.json
202+
src/BenchmarkDotNet/Disassemblers/net461/*

‎LazyCache.Benchmarks/BenchmarkConfig.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using BenchmarkDotNet.Analysers;
2+
using BenchmarkDotNet.Columns;
3+
using BenchmarkDotNet.Configs;
4+
using BenchmarkDotNet.Diagnosers;
5+
using BenchmarkDotNet.Jobs;
6+
using BenchmarkDotNet.Loggers;
7+
using BenchmarkDotNet.Reports;
8+
using Perfolizer.Horology;
9+
10+
namespace LazyCache.Benchmarks
11+
{
12+
public class BenchmarkConfig: ManualConfig
13+
{
14+
public BenchmarkConfig()
15+
=> AddJob(Job.ShortRun)
16+
.AddDiagnoser(MemoryDiagnoser.Default)
17+
.AddLogger(new ConsoleLogger())
18+
.AddColumn(TargetMethodColumn.Method)
19+
.AddAnalyser(EnvironmentAnalyser.Default)
20+
.WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond));
21+
}
22+
}

‎LazyCache.Benchmarks/ComplexObject.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace LazyCache.Benchmarks
2+
{
3+
public class ComplexObject
4+
{
5+
public string String { get; set; } = string.Empty;
6+
public int Int { get; set; } = default;
7+
}
8+
}

‎LazyCache.Benchmarks/LazyCache.Benchmarks.csproj

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>netcoreapp3.1</TargetFramework>
6+
<Optimize>true</Optimize>
7+
<Configuration>Release</Configuration>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
12+
</ItemGroup>
13+
14+
<ItemGroup>
15+
<ProjectReference Include="..\LazyCache\LazyCache.csproj" />
16+
</ItemGroup>
17+
18+
</Project>

‎LazyCache.Benchmarks/MemoryCacheBenchmarks.cs

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
using System.Threading.Tasks;
2+
using BenchmarkDotNet.Attributes;
3+
using BenchmarkDotNet.Configs;
4+
using LazyCache.Providers;
5+
using Microsoft.Extensions.Caching.Memory;
6+
7+
namespace LazyCache.Benchmarks
8+
{
9+
[Config(typeof(BenchmarkConfig))]
10+
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
11+
public class MemoryCacheBenchmarks
12+
{
13+
public const string CacheKey = nameof(CacheKey);
14+
15+
public IMemoryCache MemCache;
16+
public IMemoryCache PopulatedMemCache;
17+
public IAppCache AppCache;
18+
public IAppCache PopulatedAppCache;
19+
public ComplexObject ComplexObject;
20+
21+
[GlobalSetup]
22+
public void Setup()
23+
{
24+
ComplexObject = new ComplexObject();
25+
26+
MemCache = new MemoryCache(new MemoryCacheOptions());
27+
PopulatedMemCache = new MemoryCache(new MemoryCacheOptions());
28+
29+
AppCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
30+
PopulatedAppCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
31+
32+
PopulatedAppCache.Add(CacheKey, ComplexObject);
33+
PopulatedMemCache.Set(CacheKey, ComplexObject);
34+
}
35+
36+
[GlobalCleanup]
37+
public void Cleanup() => MemCache.Dispose();
38+
39+
/*
40+
*
41+
* Benchmark Cache Initialization
42+
*
43+
*/
44+
45+
[Benchmark(Baseline = true), BenchmarkCategory("Init")]
46+
public MemoryCache DotNetMemoryCache_Init() => new MemoryCache(new MemoryCacheOptions());
47+
48+
[Benchmark, BenchmarkCategory("Init")]
49+
public CachingService LazyCache_Init() => new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
50+
51+
/*
52+
*
53+
* Benchmark Add Methods
54+
*
55+
*/
56+
57+
[Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Add))]
58+
public void DotNetMemoryCache_Set() => MemCache.Set(CacheKey, ComplexObject);
59+
60+
[Benchmark, BenchmarkCategory(nameof(IAppCache.Add))]
61+
public void LazyCache_Set() => AppCache.Add(CacheKey, ComplexObject);
62+
63+
/*
64+
*
65+
* Benchmark Get Methods With a Cache Miss
66+
*
67+
*/
68+
69+
[Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Get) + "_Miss")]
70+
public ComplexObject DotNetMemoryCache_Get_Miss() => MemCache.Get<ComplexObject>(CacheKey);
71+
72+
[Benchmark, BenchmarkCategory(nameof(IAppCache.Get) + "_Miss")]
73+
public ComplexObject LazyCache_Get_Miss() => AppCache.Get<ComplexObject>(CacheKey);
74+
75+
/*
76+
*
77+
* Benchmark Get Methods With a Cache Hit
78+
*
79+
*/
80+
81+
[Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.Get) + "_Hit")]
82+
public ComplexObject DotNetMemoryCache_Get_Hit() => PopulatedMemCache.Get<ComplexObject>(CacheKey);
83+
84+
[Benchmark, BenchmarkCategory(nameof(IAppCache.Get) + "_Hit")]
85+
public ComplexObject LazyCache_Get_Hit() => PopulatedAppCache.Get<ComplexObject>(CacheKey);
86+
87+
/*
88+
*
89+
* Benchmark GetOrAdd Methods With Cache Miss
90+
*
91+
*/
92+
93+
[Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Miss")]
94+
public ComplexObject DotNetMemoryCache_GetOrAdd_Miss() => MemCache.GetOrCreate(CacheKey, entry => ComplexObject);
95+
96+
[Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Miss")]
97+
public ComplexObject LazyCache_GetOrAdd_Miss() => AppCache.GetOrAdd(CacheKey, entry => ComplexObject);
98+
99+
/*
100+
*
101+
* Benchmark GetOrAdd Methods With Cache Hit
102+
*
103+
*/
104+
105+
[Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Hit")]
106+
public ComplexObject DotNetMemoryCache_GetOrAdd_Hit() => PopulatedMemCache.GetOrCreate(CacheKey, entry => ComplexObject);
107+
108+
[Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAdd) + "_Hit")]
109+
public ComplexObject LazyCache_GetOrAdd_Hit() => PopulatedAppCache.GetOrAdd(CacheKey, entry => ComplexObject);
110+
111+
/*
112+
*
113+
* Benchmark GetOrAddAsync Methods With Cache Miss
114+
*
115+
*/
116+
117+
118+
[Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Miss")]
119+
public Task<ComplexObject> DotNetMemoryCache_GetOrAddAsync_Miss() => MemCache.GetOrCreateAsync(CacheKey, entry => Task.FromResult(ComplexObject));
120+
121+
[Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Miss")]
122+
public Task<ComplexObject> LazyCache_GetOrAddAsync_Miss() => AppCache.GetOrAddAsync(CacheKey, entry => Task.FromResult(ComplexObject));
123+
124+
/*
125+
*
126+
* Benchmark GetOrAddAsync Methods With Cache Hit
127+
*
128+
*/
129+
130+
[Benchmark(Baseline = true), BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Hit")]
131+
public Task<ComplexObject> DotNetMemoryCache_GetOrAddAsync_Hit() => PopulatedMemCache.GetOrCreateAsync(CacheKey, entry => Task.FromResult(ComplexObject));
132+
133+
[Benchmark, BenchmarkCategory(nameof(IAppCache.GetOrAddAsync) + "_Hit")]
134+
public Task<ComplexObject> LazyCache_GetOrAddAsync_Hit() => PopulatedAppCache.GetOrAddAsync(CacheKey, entry => Task.FromResult(ComplexObject));
135+
}
136+
}

‎LazyCache.Benchmarks/MemoryCacheBenchmarksRealLifeScenarios.cs

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
using System;
2+
using System.Management;
3+
using System.Threading.Tasks;
4+
using BenchmarkDotNet.Attributes;
5+
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.Reports;
7+
using LazyCache.Providers;
8+
using Microsoft.Extensions.Caching.Memory;
9+
10+
namespace LazyCache.Benchmarks
11+
{
12+
[Config(typeof(BenchmarkConfig))]
13+
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
14+
public class MemoryCacheBenchmarksRealLifeScenarios
15+
{
16+
public const string CacheKey = nameof(CacheKey);
17+
18+
public ComplexObject ComplexObject1;
19+
public ComplexObject ComplexObject2;
20+
public ComplexObject ComplexObject3;
21+
public ComplexObject ComplexObject4;
22+
public ComplexObject ComplexObject5;
23+
24+
// Trying not to introduce artificial allocations below - just measuring what the library itself needs
25+
[GlobalSetup]
26+
public void Setup()
27+
{
28+
ComplexObject1 = new ComplexObject();
29+
ComplexObject2 = new ComplexObject();
30+
ComplexObject3 = new ComplexObject();
31+
ComplexObject4 = new ComplexObject();
32+
ComplexObject5 = new ComplexObject();
33+
}
34+
35+
[Benchmark]
36+
public ComplexObject Init_CRUD()
37+
{
38+
var cache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions()))) as IAppCache;
39+
40+
cache.Add(CacheKey, ComplexObject1);
41+
42+
var obj = cache.Get<ComplexObject>(CacheKey);
43+
44+
obj.Int = 256;
45+
cache.Add(CacheKey, obj);
46+
47+
cache.Remove(CacheKey);
48+
49+
return obj;
50+
}
51+
52+
// Benchmark memory usage to ensure only a single instance of the object is created
53+
// Due to the nature of AsyncLazy, this test should also only take the the time it takes to create
54+
// one instance of the object.
55+
[Benchmark]
56+
public async Task<byte[]> Several_initializations_of_1Mb_object_with_200ms_delay()
57+
{
58+
var cache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions()))) as IAppCache;
59+
60+
Task AddByteArrayToCache() =>
61+
cache.GetOrAddAsync(CacheKey, async () =>
62+
{
63+
await Task.Delay(200);
64+
return await Task.FromResult(new byte[1024 * 1024]); // 1Mb
65+
});
66+
67+
// Even though the second and third init attempts are later, this whole operation should still take the time of the first
68+
var creationTask1 = AddByteArrayToCache(); // initialization attempt, or 200ms
69+
var creationTask2 = Delayed(50, AddByteArrayToCache);
70+
var creationTask3 = Delayed(50, AddByteArrayToCache);
71+
72+
await Task.WhenAll(creationTask1, creationTask2, creationTask3);
73+
74+
return cache.Get<byte[]>(CacheKey);
75+
}
76+
77+
private async Task Delayed(int ms, Func<Task> action)
78+
{
79+
await Task.Delay(ms);
80+
await action();
81+
}
82+
}
83+
}

‎LazyCache.Benchmarks/Program.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
using BenchmarkDotNet.Running;
2+
3+
namespace LazyCache.Benchmarks
4+
{
5+
public static class Program
6+
{
7+
public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
8+
}
9+
}

‎LazyCache.Benchmarks/Properties/launchSettings.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"profiles": {
3+
"LazyCache.Benchmarks": {
4+
"commandName": "Project"
5+
}
6+
}
7+
}

‎LazyCache.Benchmarks/README.md

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# LazyCache.Benchmarks
2+
This project is dedicated towards benchmarking (using [BenchmarkDotNet](https://benchmarkdotnet.org/index.html)) the basic functionality of LazyCache such that contributors and maintainers can verify the efficacy of changes towards the project - for better or for worse.
3+
4+
## Note to readers
5+
While it is always a good idea to understand performance of your third party libraries, it is rare that you will be concerned with performance on the scale of nanoseconds such that this library operates on. Be wary of premature optimization.
6+
7+
# How to run
8+
- Ensure you have the requisite dotnet SDKs found in _LazyCache.Benchmarks.csproj_
9+
- Clone the project
10+
- Open your favorite terminal, navigate to the Benchmark Project
11+
- `dotnet run -c Release`
12+
- Pick your desired benchmark suite via numeric entry
13+
14+
If you are interested in benchmarking a specific method (after making changes to it, for instance), you can conveniently filter down to one specific benchmark, e.g. `dotnet run -c Release -- -f *Get` will only run the benchmarks for `IAppCache.Get` implementations, likewise with `*GetOrAddAsync`, or other methods.
15+
16+
# Contributing
17+
If you have ideas for one or more benchmarks not covered here, please add an issue describing what you would like to see. Pull requests are always welcome!
18+
19+
# Benchmark Types
20+
There are two types of benchmarks available.
21+
22+
## Basics
23+
The basic benchmarks are small and laser-focused on testing individual aspects of LazyCache. This suite of benchmarks uses the out-of-the-box MemoryCache from dotnet [seen here](https://github.com/dotnet/runtime/blob/master/src/libraries/Microsoft.Extensions.Caching.Memory/src/) as a baseline, to demonstrate the "cost" of LazyCache in comparison.
24+
25+
## Integration
26+
These benchmarks are designed to showcase full use-cases of LazyCache by chaining together various operations. As an example, with the Memory Diagnoser from BenchmarkDotNet, we can verify that concurrent calls to initialize a cache item correctly spin up one instance of said item, with the subsequent calls awaiting its result.
27+
28+
### Gotchas
29+
Remember that BenchmarkDotNet dutifully monitors allocations inside the benchmark method, and _only_ the method. At the time of writing, the default instance of the MemoryCacheProvider is static, and allocations into this cache will **not** be monitored by BenchmarkDotNet. For all benchmarks, please ensure you are creating new instances of the Service, Provider, and backing Cache.
30+
31+
# Benchmarks
32+
33+
```
34+
// * Summary *
35+
BenchmarkDotNet=v0.12.1, OS=Windows 10.0.18362.1082 (1903/May2019Update/19H1)
36+
AMD Ryzen 9 3900X, 1 CPU, 24 logical and 12 physical cores
37+
.NET Core SDK=5.0.100-preview.7.20366.6
38+
[Host] : .NET Core 3.1.7 (CoreCLR 4.700.20.36602, CoreFX 4.700.20.37001), X64 RyuJIT
39+
ShortRun : .NET Core 3.1.7 (CoreCLR 4.700.20.36602, CoreFX 4.700.20.37001), X64 RyuJIT
40+
41+
Job=ShortRun IterationCount=3 LaunchCount=1
42+
WarmupCount=3
43+
```
44+
| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen 0 | Gen 1 | Gen 2 | Allocated |
45+
|------------------------------------- |-----------:|------------:|---------:|------:|--------:|-------:|-------:|-------:|----------:|
46+
| DotNetMemoryCache_Init | 1,814.2 ns | 1,080.95 ns | 59.25 ns | 1.00 | 0.00 | 0.1850 | 0.0916 | 0.0019 | 1560 B |
47+
| LazyCache_Init | 3,265.5 ns | 599.75 ns | 32.87 ns | 1.80 | 0.07 | 0.3090 | 0.1526 | - | 2600 B |
48+
| | | | | | | | | | |
49+
| DotNetMemoryCache_Set | 504.1 ns | 42.38 ns | 2.32 ns | 1.00 | 0.00 | 0.0496 | - | - | 416 B |
50+
| LazyCache_Set | 841.6 ns | 172.51 ns | 9.46 ns | 1.67 | 0.02 | 0.0801 | - | - | 672 B |
51+
| | | | | | | | | | |
52+
| DotNetMemoryCache_Get_Miss | 201.1 ns | 3.54 ns | 0.19 ns | 1.00 | 0.00 | - | - | - | - |
53+
| LazyCache_Get_Miss | 241.1 ns | 13.94 ns | 0.76 ns | 1.20 | 0.00 | - | - | - | - |
54+
| | | | | | | | | | |
55+
| DotNetMemoryCache_Get_Hit | 242.2 ns | 28.93 ns | 1.59 ns | 1.00 | 0.00 | - | - | - | - |
56+
| LazyCache_Get_Hit | 280.4 ns | 10.45 ns | 0.57 ns | 1.16 | 0.01 | - | - | - | - |
57+
| | | | | | | | | | |
58+
| DotNetMemoryCache_GetOrAdd_Miss | 269.9 ns | 6.57 ns | 0.36 ns | 1.00 | 0.00 | 0.0076 | - | - | 64 B |
59+
| LazyCache_GetOrAdd_Miss | 368.5 ns | 60.35 ns | 3.31 ns | 1.37 | 0.01 | 0.0191 | - | - | 160 B |
60+
| | | | | | | | | | |
61+
| DotNetMemoryCache_GetOrAdd_Hit | 269.1 ns | 4.48 ns | 0.25 ns | 1.00 | 0.00 | 0.0076 | - | - | 64 B |
62+
| LazyCache_GetOrAdd_Hit | 377.1 ns | 10.57 ns | 0.58 ns | 1.40 | 0.00 | 0.0191 | - | - | 160 B |
63+
| | | | | | | | | | |
64+
| DotNetMemoryCache_GetOrAddAsync_Miss | 312.7 ns | 53.05 ns | 2.91 ns | 1.00 | 0.00 | 0.0162 | - | - | 136 B |
65+
| LazyCache_GetOrAddAsync_Miss | 507.5 ns | 33.96 ns | 1.86 ns | 1.62 | 0.02 | 0.0362 | - | - | 304 B |
66+
| | | | | | | | | | |
67+
| DotNetMemoryCache_GetOrAddAsync_Hit | 314.5 ns | 65.34 ns | 3.58 ns | 1.00 | 0.00 | 0.0162 | - | - | 136 B |
68+
| LazyCache_GetOrAddAsync_Hit | 535.9 ns | 47.83 ns | 2.62 ns | 1.70 | 0.03 | 0.0448 | - | - | 376 B |
69+
70+
| Method | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated |
71+
|------------------------------------------------------- |-----------------:|----------------:|----------------:|-------:|-------:|-------:|-----------:|
72+
| Init_CRUD | 5,115.1 ns | 991.0 ns | 54.32 ns | 0.4730 | 0.2365 | 0.0076 | 3.9 KB |
73+
| Several_initializations_of_1Mb_object_with_200ms_delay | 207,329,988.9 ns | 31,342,899.9 ns | 1,718,010.11 ns | - | - | - | 1031.75 KB |

0 commit comments

Comments
 (0)