Add performace improvements

README.md

@@ -21,17 +21,20 @@ _In case you will use `false` bellow configs what describe password encryption n
 **Setting define ip address on which mongo db is listening** (_not change is needed_)<br />
 `<add key="Mongo-Connection-String" value="mongodb://127.0.0.1:27117" />`
 
-**Setting define connection string to Sql server**<br />
+**Setting define connection string to Sql server** (_Always have pooling disabled `Pooling=false;`_)<br />
 `<add key="Sql-Connection-String" value="Server=[SQL-SERVER];Database=[DATABASE-NAME];" />`
 - `[SQL-SERVER]` - IP address or FQDN of SQL server
 - `[DATABASE-NAME]` - name of database where exporter will save data
 
 **Setting defines username|userId which will be used for SQL server connection**<br />
 `<add key="Sql-User-Id" value="" />`
 
-**Setting defines password which will be used for SQL server connection** (_have to be encrypted_) <br/>
+**Setting defines password which will be used for SQL server connection** (_have to be encrypted_) <br />
 `<add key="Sql-User-Password" value="" />`
 
+**Setting defines how many records will be handled in single transaction**<br />
+`<add key="Sql-Batch-Size" value=""/>`
+
 **Setting defines if Filesystem exporter will be used or not**<br />
 `<add key="Export-To-FS" value="true|false" />`
 

src/MalikP.Ubiquiti.DatabaseExporter.Core/Extensions/ObjectExtensions.cs

@@ -0,0 +1,41 @@
+// Copyright (c) 2018 Peter M.
+// 
+// File: ObjectExtensions.cs 
+// Company: MalikP.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+// 
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+using MalikP.Ubiquiti.DatabaseExporter.Core.Interfaces;
+
+namespace MalikP.Ubiquiti.DatabaseExporter.Core.Extensions
+{
+
+    public static class ObjectExtensions
+    {
+        public static void TryInitialize<T>(this object source, T initializationInstance)
+        {
+            var casted = source as IInitializable;
+            if (casted != null)
+            {
+                casted.Initialize<T>(initializationInstance);
+            }
+        }
+    }
+}

src/MalikP.Ubiquiti.DatabaseExporter.Core/Interfaces/IInitializable.cs

@@ -0,0 +1,31 @@
+// Copyright (c) 2018 Peter M.
+// 
+// File: IInitializable.cs 
+// Company: MalikP.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+// 
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+namespace MalikP.Ubiquiti.DatabaseExporter.Core.Interfaces
+{
+    public interface IInitializable
+    {
+        IInitializable Initialize<T>(T initializationInstance);
+    }
+}

src/MalikP.Ubiquiti.DatabaseExporter.Core/MalikP.Ubiquiti.DatabaseExporter.Core.csproj

@@ -42,7 +42,8 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="SecureStringExtensions.cs" />
+    <Compile Include="Extensions\ObjectExtensions.cs" />
+    <Compile Include="Interfaces\IInitializable.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

src/MalikP.Ubiquiti.DatabaseExporter.Core/Properties/AssemblyInfo.cs

@@ -55,5 +55,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.1.0.0")]
-[assembly: AssemblyFileVersion("1.1.0.0")]
+[assembly: AssemblyVersion("1.2.0.0")]
+[assembly: AssemblyFileVersion("1.2.0.0")]

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/AbstractDatabaseExecutor.cs

@@ -27,6 +27,7 @@
 using System.Collections.Generic;
 using System.Data;
 using System.Data.SqlClient;
+using System.Linq;
 
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core
 {
@@ -60,6 +61,26 @@ protected virtual IDbCommand GetCommand(AbstractCommandCreator abstractCommandCr
             return abstractCommandCreator?.CreateCommand(connection);
         }
 
+        protected void ExecuteNonQuery(IEnumerable<IDbCommand> commands, IDbTransaction transaction)
+        {
+            if (commands == null || !commands.Any())
+            {
+                return;
+            }
+
+            foreach (IDbCommand command in commands)
+            {
+                command.ExecuteNonQuery();
+            }
+
+            transaction.Commit();
+
+            foreach (IDbCommand command in commands)
+            {
+                command.Dispose();
+            }
+        }
+
         protected int ExecuteNonQuery(IDbCommand command)
         {
             if (command == null)
@@ -77,7 +98,7 @@ protected int ExecuteNonQuery(IDbCommand command)
         protected IEnumerable<T> ExecuteReaderWithManyResults<T>(IDbCommand command)
             where T : class, IMapable, new()
         {
-            using (command.Connection)
+            using (command?.Connection)
             using (command)
             using (var reader = command.ExecuteReader())
             {
@@ -91,7 +112,7 @@ protected IEnumerable<T> ExecuteReaderWithManyResults<T>(IDbCommand command)
         protected T ExecuteReaderWithSingleResult<T>(IDbCommand command, T entity = null)
            where T : class, IMapable, new()
         {
-            using (command.Connection)
+            using (command?.Connection)
             using (command)
             using (var reader = command.ExecuteReader())
             {

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Abstract_CommandCreator_CheckId.cs

@@ -40,12 +40,12 @@ protected Abstract_CommandCreator_CheckId(string schema, string tableName, strin
 
         protected override string CreateCommandText()
         {
-            return $"select {TableName}.JsonDataId from {TableName} where JsonDataId = @idToCheck";
+            return $"select {TableName}.JsonDataId from {TableName} where JsonDataId IN";
         }
 
         protected override void SetupCommand(SqlCommand command)
         {
-            command.Parameters.AddWithValue("@idToCheck", IdToCheck);
+            command.CommandText = $"{CreateCommandText()}({IdToCheck})";
             command.CommandType = CommandType.Text;
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Abstract_CommandCreator_WriteId.cs

@@ -22,23 +22,26 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
 using System.Data;
 using System.Data.SqlClient;
 
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators
 {
     public abstract class Abstract_CommandCreator_WriteId : AbstractCommandCreator
     {
-        protected Abstract_CommandCreator_WriteId(string schema, string tableName, string jsonDataId, string jsonData)
+        private readonly Dictionary<string, string> _documentDictionary;
+
+        protected Abstract_CommandCreator_WriteId(string schema, string tableName, Dictionary<string, string> documentDictionary)
         {
             TableName = $"[{schema}].[{tableName}]";
-            JsonDataId = jsonDataId;
-            JsonData = jsonData;
+            _documentDictionary = documentDictionary;
+            Commands = new List<IDbCommand>();
         }
 
         protected string TableName { get; }
-        protected string JsonDataId { get; }
-        protected string JsonData { get; }
+
+        public List<IDbCommand> Commands { get; }
 
         protected override string CreateCommandText()
         {
@@ -51,9 +54,18 @@ protected override string CreateCommandText()
 
         protected override void SetupCommand(SqlCommand command)
         {
-            command.Parameters.AddWithValue("@jsonData", JsonData);
-            command.Parameters.AddWithValue("@jsonDataId", JsonDataId);
-            command.CommandType = CommandType.Text;
+            var connection = command.Connection;
+            var transaction = connection.BeginTransaction();
+
+            foreach (var item in _documentDictionary)
+            {
+                var newCommand = new SqlCommand(CreateCommandText(), connection);
+                Commands.Add(newCommand);
+                newCommand.Parameters.AddWithValue("@jsonData", item.Value);
+                newCommand.Parameters.AddWithValue("@jsonDataId", item.Key);
+                newCommand.CommandType = CommandType.Text;
+                newCommand.Transaction = transaction;
+            }
         }
     }
 }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_Ace.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public abstract class CommandCreator_WriteId_Ace : Abstract_CommandCreator_WriteId
     {
-        protected CommandCreator_WriteId_Ace(string tableName, string jsonDataId, string jsonData)
-            : base("ace", tableName, jsonDataId, jsonData)
+        protected CommandCreator_WriteId_Ace(string tableName, Dictionary<string, string> documentDictionary)
+            : base("ace", tableName, documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_account.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_account : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_account(string jsonDataId, string jsonData)
-            : base("account", jsonDataId, jsonData)
+        public CommandCreator_WriteId_account(Dictionary<string, string> documentDictionary)
+            : base("account", documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_admin.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_admin : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_admin(string jsonDataId, string jsonData)
-            : base("admin", jsonDataId, jsonData)
+        public CommandCreator_WriteId_admin(Dictionary<string, string> documentDictionary)
+            : base("admin", documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_alarm.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_alarm : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_alarm(string jsonDataId, string jsonData)
-            : base("alarm", jsonDataId, jsonData)
+        public CommandCreator_WriteId_alarm(Dictionary<string, string> documentDictionary)
+            : base("alarm", documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_broadcastgroup.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_broadcastgroup : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_broadcastgroup(string jsonDataId, string jsonData)
-            : base("broadcastgroup", jsonDataId, jsonData)
+        public CommandCreator_WriteId_broadcastgroup(Dictionary<string, string> documentDictionary)
+            : base("broadcastgroup", documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_dashboard.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_dashboard : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_dashboard(string jsonDataId, string jsonData)
-            : base("dashboard", jsonDataId, jsonData)
+        public CommandCreator_WriteId_dashboard(Dictionary<string, string> documentDictionary)
+            : base("dashboard", documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_device.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_device : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_device(string jsonDataId, string jsonData)
-            : base("device", jsonDataId, jsonData)
+        public CommandCreator_WriteId_device(Dictionary<string, string> documentDictionary)
+            : base("device", documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_dhcpoption.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_dhcpoption : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_dhcpoption(string jsonDataId, string jsonData)
-            : base("dhcpoption", jsonDataId, jsonData)
+        public CommandCreator_WriteId_dhcpoption(Dictionary<string, string> documentDictionary)
+            : base("dhcpoption", documentDictionary)
         {
         }
     }

src/MalikP.Ubiquiti.DatabaseExporter.Data/Core/CommandCreators/Ace/CommandCreator_WriteId_dpiapp.cs

@@ -22,12 +22,14 @@
 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+using System.Collections.Generic;
+
 namespace MalikP.Ubiquiti.DatabaseExporter.Data.Core.CommandCreators.Ace
 {
     public class CommandCreator_WriteId_dpiapp : CommandCreator_WriteId_Ace
     {
-        public CommandCreator_WriteId_dpiapp(string jsonDataId, string jsonData)
-            : base("dpiapp", jsonDataId, jsonData)
+        public CommandCreator_WriteId_dpiapp(Dictionary<string, string> documentDictionary)
+            : base("dpiapp", documentDictionary)
         {
         }
     }