From 1527699084bf651213eb132f2619518f29e33cfb Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Wed, 19 Apr 2017 14:59:03 -0700 Subject: [PATCH] Add Benchmark project. This compares the current code to MySql.Data 6.9.9. Currently only built for the full framework in order to use "extern alias" to reference both MySQL client libraries. --- .ci/build.ps1 | 2 +- .ci/test.ps1 | 9 + .gitignore | 1 + appveyor.yml | 1 + tests/Benchmark/App.config | 23 +++ tests/Benchmark/Benchmark.csproj | 173 ++++++++++++++++++ tests/Benchmark/Benchmark.sln | 28 +++ tests/Benchmark/Program.cs | 198 +++++++++++++++++++++ tests/Benchmark/Properties/AssemblyInfo.cs | 12 ++ tests/Benchmark/packages.config | 51 ++++++ 10 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 tests/Benchmark/App.config create mode 100644 tests/Benchmark/Benchmark.csproj create mode 100644 tests/Benchmark/Benchmark.sln create mode 100644 tests/Benchmark/Program.cs create mode 100644 tests/Benchmark/Properties/AssemblyInfo.cs create mode 100644 tests/Benchmark/packages.config diff --git a/.ci/build.ps1 b/.ci/build.ps1 index 42c55a4a4..26865cf83 100644 --- a/.ci/build.ps1 +++ b/.ci/build.ps1 @@ -61,7 +61,7 @@ if (!(Test-Path $buildFolder)) { } } -$dotnetVersion = "1.0.0" +$dotnetVersion = "1.0.3" $dotnetChannel = "rel-1.0.0" $dotnetSharedRuntimeVersion = "1.1.1" $dotnetSharedRuntimeChannel = "rel-1.0.0" diff --git a/.ci/test.ps1 b/.ci/test.ps1 index 68ce31090..d95b6e9a7 100644 --- a/.ci/test.ps1 +++ b/.ci/test.ps1 @@ -51,3 +51,12 @@ dotnet test tests\SideBySide\SideBySide.csproj -c Baseline if ($LASTEXITCODE -ne 0){ exit $LASTEXITCODE; } + +echo "Building Benchmark" + +& "nuget.exe" restore tests\Benchmark\Benchmark.sln +& "msbuild.exe" tests\Benchmark\Benchmark.sln /p:Configuration=Release +if ($LASTEXITCODE -ne 0){ + exit $LASTEXITCODE; +} +.\tests\Benchmark\bin\Release\Benchmark.exe diff --git a/.gitignore b/.gitignore index b5979467d..01be36584 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # ignore output folders +BenchmarkDotNet.Artifacts/ bin/ build/ obj/ diff --git a/appveyor.yml b/appveyor.yml index 6b0be93f2..6c56a1543 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,3 +1,4 @@ +image: Visual Studio 2017 services: - mysql build_script: diff --git a/tests/Benchmark/App.config b/tests/Benchmark/App.config new file mode 100644 index 000000000..f6f3351da --- /dev/null +++ b/tests/Benchmark/App.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/Benchmark/Benchmark.csproj b/tests/Benchmark/Benchmark.csproj new file mode 100644 index 000000000..c05294d84 --- /dev/null +++ b/tests/Benchmark/Benchmark.csproj @@ -0,0 +1,173 @@ + + + + + Debug + AnyCPU + {102C06D1-ADB7-4661-9EC5-F6460BFEDB2A} + Exe + Properties + Benchmark + Benchmark + v4.6.1 + 512 + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + false + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + false + + + + packages\BenchmarkDotNet.0.10.3\lib\net45\BenchmarkDotNet.dll + True + + + packages\BenchmarkDotNet.Core.0.10.3\lib\net45\BenchmarkDotNet.Core.dll + True + + + packages\BenchmarkDotNet.Toolchains.Roslyn.0.10.3\lib\net45\BenchmarkDotNet.Toolchains.Roslyn.dll + True + + + packages\Microsoft.CodeAnalysis.Common.1.3.2\lib\net45\Microsoft.CodeAnalysis.dll + True + + + packages\Microsoft.CodeAnalysis.CSharp.1.3.2\lib\net45\Microsoft.CodeAnalysis.CSharp.dll + True + + + packages\MySql.Data.6.9.9\lib\net45\MySql.Data.dll + True + MySqlData + + + + packages\System.AppContext.4.1.0\lib\net46\System.AppContext.dll + True + + + packages\System.Collections.Immutable.1.2.0\lib\portable-net45+win8+wp8+wpa81\System.Collections.Immutable.dll + True + + + + packages\System.Console.4.0.0\lib\net46\System.Console.dll + True + + + + packages\System.Diagnostics.FileVersionInfo.4.0.0\lib\net46\System.Diagnostics.FileVersionInfo.dll + True + + + packages\System.Diagnostics.StackTrace.4.0.1\lib\net46\System.Diagnostics.StackTrace.dll + True + + + packages\System.IO.FileSystem.4.0.1\lib\net46\System.IO.FileSystem.dll + True + + + packages\System.IO.FileSystem.Primitives.4.0.1\lib\net46\System.IO.FileSystem.Primitives.dll + True + + + + packages\System.Reflection.Metadata.1.3.0\lib\portable-net45+win8\System.Reflection.Metadata.dll + True + + + packages\System.Security.Cryptography.Algorithms.4.2.0\lib\net461\System.Security.Cryptography.Algorithms.dll + True + + + packages\System.Security.Cryptography.Encoding.4.0.0\lib\net46\System.Security.Cryptography.Encoding.dll + True + + + packages\System.Security.Cryptography.Primitives.4.0.0\lib\net46\System.Security.Cryptography.Primitives.dll + True + + + packages\System.Security.Cryptography.X509Certificates.4.1.0\lib\net461\System.Security.Cryptography.X509Certificates.dll + True + + + packages\System.Text.Encoding.CodePages.4.0.1\lib\net46\System.Text.Encoding.CodePages.dll + True + + + packages\System.Threading.Tasks.Extensions.4.3.0\lib\portable-net45+win8+wp8+wpa81\System.Threading.Tasks.Extensions.dll + True + + + packages\System.Threading.Thread.4.0.0\lib\net46\System.Threading.Thread.dll + True + + + + + + + + + packages\System.Xml.XmlDocument.4.0.1\lib\net46\System.Xml.XmlDocument.dll + True + + + packages\System.Xml.XPath.4.0.1\lib\net46\System.Xml.XPath.dll + True + + + packages\System.Xml.XPath.XDocument.4.0.1\lib\net46\System.Xml.XPath.XDocument.dll + True + + + + + + + + + + + + + + + + + {6b4fea15-4504-4bf7-b157-cef9b60cfe13} + MySqlConnector + MySqlConnector + + + + + \ No newline at end of file diff --git a/tests/Benchmark/Benchmark.sln b/tests/Benchmark/Benchmark.sln new file mode 100644 index 000000000..882832779 --- /dev/null +++ b/tests/Benchmark/Benchmark.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26403.7 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "Benchmark.csproj", "{102C06D1-ADB7-4661-9EC5-F6460BFEDB2A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MySqlConnector", "..\..\src\MySqlConnector\MySqlConnector.csproj", "{6B4FEA15-4504-4BF7-B157-CEF9B60CFE13}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {102C06D1-ADB7-4661-9EC5-F6460BFEDB2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {102C06D1-ADB7-4661-9EC5-F6460BFEDB2A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {102C06D1-ADB7-4661-9EC5-F6460BFEDB2A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {102C06D1-ADB7-4661-9EC5-F6460BFEDB2A}.Release|Any CPU.Build.0 = Release|Any CPU + {6B4FEA15-4504-4BF7-B157-CEF9B60CFE13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6B4FEA15-4504-4BF7-B157-CEF9B60CFE13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6B4FEA15-4504-4BF7-B157-CEF9B60CFE13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6B4FEA15-4504-4BF7-B157-CEF9B60CFE13}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/tests/Benchmark/Program.cs b/tests/Benchmark/Program.cs new file mode 100644 index 000000000..1d8147d27 --- /dev/null +++ b/tests/Benchmark/Program.cs @@ -0,0 +1,198 @@ +extern alias MySqlData; +extern alias MySqlConnector; +using System; +using System.Data.Common; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; +using OldMySqlConnection = MySqlData.MySql.Data.MySqlClient.MySqlConnection; +using NewMySqlConnection = MySqlConnector.MySql.Data.MySqlClient.MySqlConnection; + +namespace Benchmark +{ + class Program + { + static void Main() + { + var customConfig = ManualConfig + .Create(DefaultConfig.Instance) + .With(JitOptimizationsValidator.FailOnError) + .With(MemoryDiagnoser.Default) + .With(StatisticColumn.AllStatistics) + .With(DefaultExporters.Csv); + + var summary = BenchmarkRunner.Run(customConfig); + Console.WriteLine(summary); + } + } + + public class MySqlClient + { + [Setup] + public void Setup() + { + using (var oldConnection = new OldMySqlConnection(s_connectionString)) + { + oldConnection.Open(); + RunSetupSql(oldConnection); + } + + using (var newConnection = new NewMySqlConnection(s_connectionString)) + { + newConnection.Open(); + RunSetupSql(newConnection); + } + + s_connectionString += ";database=benchmark"; + + m_oldConnection = new OldMySqlConnection(s_connectionString); + m_oldConnection.Open(); + + m_newConnection = new NewMySqlConnection(s_connectionString); + m_newConnection.Open(); + } + + private void RunSetupSql(DbConnection connection) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = @" +create schema if not exists benchmark; + +drop table if exists benchmark.integers; +create table benchmark.integers (value int not null primary key); +insert into benchmark.integers(value) values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20); + +drop table if exists benchmark.blobs; +create table benchmark.blobs( + rowid integer not null primary key auto_increment, + `Blob` longblob null +); +insert into benchmark.blobs(`Blob`) values(null), (@Blob1), (@Blob2);"; + + // larger blobs make the tests run much slower + AddBlobParameter(cmd, "@Blob1", 100000); + AddBlobParameter(cmd, "@Blob2", 1000000); + + cmd.ExecuteNonQuery(); + } + } + + private static void AddBlobParameter(DbCommand command, string name, int size) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = name; + + var random = new Random(size); + var value = new byte[size]; + random.NextBytes(value); + parameter.Value = value; + + command.Parameters.Add(parameter); + } + + [Benchmark] public Task OpenFromPoolOldAsync() => OpenFromPoolAsync(m_oldConnection); + [Benchmark] public void OpenFromPoolOldSync() => OpenFromPoolSync(m_oldConnection); + [Benchmark] public Task OpenFromPoolNewAsync() => OpenFromPoolAsync(m_newConnection); + [Benchmark] public void OpenFromPoolNewSync() => OpenFromPoolSync(m_newConnection); + + private static async Task OpenFromPoolAsync(DbConnection connection) + { + connection.Close(); + await connection.OpenAsync(); + } + + private static void OpenFromPoolSync(DbConnection connection) + { + connection.Close(); + connection.Open(); + } + + [Benchmark] public Task ExecuteScalarOldAsync() => ExecuteScalarAsync(m_oldConnection); + [Benchmark] public void ExecuteScalarOldSync() => ExecuteScalarSync(m_oldConnection); + [Benchmark] public Task ExecuteScalarNewAsync() => ExecuteScalarAsync(m_newConnection); + [Benchmark] public void ExecuteScalarNewSync() => ExecuteScalarSync(m_newConnection); + + private static async Task ExecuteScalarAsync(DbConnection connection) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = c_executeScalarSql; + await cmd.ExecuteScalarAsync(); + } + } + + private static void ExecuteScalarSync(DbConnection connection) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = c_executeScalarSql; + cmd.ExecuteScalar(); + } + } + + private const string c_executeScalarSql = "select max(value) from integers;"; + + [Benchmark] public Task ReadBlobsOldAsync() => ReadAllRowsAsync(m_oldConnection, c_readBlobsSql); + [Benchmark] public void ReadBlobsOldSync() => ReadAllRowsSync(m_oldConnection, c_readBlobsSql); + [Benchmark] public Task ReadBlobsNewAsync() => ReadAllRowsAsync(m_newConnection, c_readBlobsSql); + [Benchmark] public void ReadBlobsNewSync() => ReadAllRowsSync(m_newConnection, c_readBlobsSql); + + private const string c_readBlobsSql = "select `Blob` from blobs;"; + + [Benchmark] public Task ManyRowsOldAsync() => ReadAllRowsAsync(m_oldConnection, c_manyRowsSql); + [Benchmark] public void ManyRowsOldSync() => ReadAllRowsSync(m_oldConnection, c_manyRowsSql); + [Benchmark] public Task ManyRowsNewAsync() => ReadAllRowsAsync(m_newConnection, c_manyRowsSql); + [Benchmark] public void ManyRowsNewSync() => ReadAllRowsSync(m_newConnection, c_manyRowsSql); + + private const string c_manyRowsSql = "select * from integers a join integers b; select * from integers a join integers b join integers c;"; + + private static async Task ReadAllRowsAsync(DbConnection connection, string sql) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = sql; + using (var reader = await cmd.ExecuteReaderAsync()) + { + do + { + while (await reader.ReadAsync()) + { + } + } while (await reader.NextResultAsync()); + } + } + } + + private static void ReadAllRowsSync(DbConnection connection, string sql) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = sql; + using (var reader = cmd.ExecuteReader()) + { + do + { + while (reader.Read()) + { + } + } while (reader.NextResult()); + } + } + } + + // TODO: move to config file + // NOTE: Without "Connection Reset=true" here, Connector/NET doesn't reset the connection, which is buggy but 8x faster + // With "Connection Reset=true" here, Connector/NET is affected by https://bugs.mysql.com/bug.php?id=80030 and is 48x slower + // We opt for the incorrect-but-faster implementation here so the benchmarks don't take as long (and give something to aim for) + static string s_connectionString = "server=127.0.0.1;user id=mysqltest;password='test;key=\"val';port=3306;ssl mode=none;Use Affected Rows=true"; + + private OldMySqlConnection m_oldConnection; + private NewMySqlConnection m_newConnection; + } +} diff --git a/tests/Benchmark/Properties/AssemblyInfo.cs b/tests/Benchmark/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..ef0911353 --- /dev/null +++ b/tests/Benchmark/Properties/AssemblyInfo.cs @@ -0,0 +1,12 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("MySqlConnector Benchmark")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("MySqlConnector")] +[assembly: AssemblyCopyright("Copyright 2017 Bradley Grainger")] +[assembly: ComVisible(false)] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tests/Benchmark/packages.config b/tests/Benchmark/packages.config new file mode 100644 index 000000000..ac41a0697 --- /dev/null +++ b/tests/Benchmark/packages.config @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file