From 8e3e9578c18b2dba3f84549b093b0d80615b1cf7 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Tue, 9 Sep 2025 12:26:55 +0500 Subject: [PATCH 01/17] feat: Initial implementation of DbUp.ClickHouse based on ClickHouse.Driver --- README.md | 2 +- src/Directory.Build.props | 2 +- ...pportTests.VerifyBasicSupport.approved.txt | 30 ++ ...yJournalCreationIfNameChanged.approved.txt | 30 ++ ...s.VerifyVariableSubstitutions.approved.txt | 30 ++ .../NoPublicApiChanges.Run.approved.cs | 38 +- src/Tests/ClickHouseContainerFixture.cs | 34 ++ src/Tests/ClickHouseIntegrationTests.cs | 435 ++++++++++++++++++ src/Tests/ClickHouseJournalTests.cs | 111 +++++ src/Tests/ClickHouseQueryParserTests.cs | 90 ++++ src/Tests/DatabaseSupportTests.cs | 4 +- src/Tests/NoPublicApiChanges.cs | 2 +- src/Tests/Tests.csproj | 9 +- src/dbup-clickhouse.sln | 6 +- .../ClickHouseConnectionManager.cs | 41 ++ src/dbup-clickhouse/ClickHouseExtensions.cs | 33 +- src/dbup-clickhouse/ClickHouseJournal.cs | 49 +- src/dbup-clickhouse/ClickHouseObjectParser.cs | 9 + src/dbup-clickhouse/ClickHousePreprocessor.cs | 12 + src/dbup-clickhouse/ClickHouseQueryParser.cs | 177 +++++++ .../ClickHouseScriptExecutor.cs | 46 ++ src/dbup-clickhouse/dbup-clickhouse.csproj | 3 +- start-clickhouse.ps1 | 55 +++ 23 files changed, 1215 insertions(+), 33 deletions(-) create mode 100644 src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyBasicSupport.approved.txt create mode 100644 src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyJournalCreationIfNameChanged.approved.txt create mode 100644 src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyVariableSubstitutions.approved.txt create mode 100644 src/Tests/ClickHouseContainerFixture.cs create mode 100644 src/Tests/ClickHouseIntegrationTests.cs create mode 100644 src/Tests/ClickHouseJournalTests.cs create mode 100644 src/Tests/ClickHouseQueryParserTests.cs create mode 100644 src/dbup-clickhouse/ClickHouseConnectionManager.cs create mode 100644 src/dbup-clickhouse/ClickHouseObjectParser.cs create mode 100644 src/dbup-clickhouse/ClickHousePreprocessor.cs create mode 100644 src/dbup-clickhouse/ClickHouseQueryParser.cs create mode 100644 src/dbup-clickhouse/ClickHouseScriptExecutor.cs create mode 100644 start-clickhouse.ps1 diff --git a/README.md b/README.md index ccae84e..59b2abf 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Prerelease](https://img.shields.io/nuget/vpre/dbup-clickhouse?color=orange&label=prerelease)](https://www.nuget.org/packages/dbup-clickhouse) # DbUp ClickHouse support -DbUp is a .NET library that helps you to deploy changes to SQL Server databases. It tracks which SQL scripts have been run already, and runs the change scripts that are needed to get your database up to date. +DbUp is a .NET library that helps you to deploy changes to databases. It tracks which SQL scripts have been run already and runs the change scripts that are needed to get your database up to date. This package adds ClickHouse support. ## Getting Help To learn more about DbUp check out the [documentation](https://dbup.readthedocs.io/en/latest/) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 814c271..011b551 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -6,7 +6,7 @@ https://github.com/DbUp/dbup-clickhouse/releases https://dbup.github.io MIT - https://github.com/dbup-clickhouse/dbup-clickhouse.git + https://github.com/DbUp/dbup-clickhouse.git latest true diff --git a/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyBasicSupport.approved.txt b/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyBasicSupport.approved.txt new file mode 100644 index 0000000..1f3453b --- /dev/null +++ b/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyBasicSupport.approved.txt @@ -0,0 +1,30 @@ +DB Operation: Open connection +Info: Beginning database upgrade +Info: Checking whether journal table exists +DB Operation: Execute scalar command: EXISTS TABLE schemaversions +DB Operation: Dispose command +Info: Journal table does not exist +Info: Executing Database Server script 'Script0001.sql' +Info: Checking whether journal table exists +DB Operation: Execute scalar command: EXISTS TABLE schemaversions +DB Operation: Dispose command +Info: Creating the `schemaversions` table +DB Operation: Execute non query command: CREATE TABLE `schemaversions` +( + ScriptName String, + Applied DateTime +) +ENGINE = MergeTree() +ORDER BY (ScriptName) +DB Operation: Dispose command +Info: The `schemaversions` table has been created +DB Operation: Execute non query command: script1contents +DB Operation: Dispose command +DB Operation: Create parameter +Info: DB Operation: Add parameter to command: scriptName=Script0001.sql +DB Operation: Create parameter +Info: DB Operation: Add parameter to command: applied= +DB Operation: Execute non query command: INSERT INTO `schemaversions` (ScriptName, Applied) VALUES (@scriptName, @applied) +DB Operation: Dispose command +Info: Upgrade successful +DB Operation: Dispose connection diff --git a/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyJournalCreationIfNameChanged.approved.txt b/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyJournalCreationIfNameChanged.approved.txt new file mode 100644 index 0000000..2fe7b94 --- /dev/null +++ b/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyJournalCreationIfNameChanged.approved.txt @@ -0,0 +1,30 @@ +DB Operation: Open connection +Info: Beginning database upgrade +Info: Checking whether journal table exists +DB Operation: Execute scalar command: EXISTS TABLE `test`.`TestSchemaVersions` +DB Operation: Dispose command +Info: Journal table does not exist +Info: Executing Database Server script 'Script0001.sql' +Info: Checking whether journal table exists +DB Operation: Execute scalar command: EXISTS TABLE `test`.`TestSchemaVersions` +DB Operation: Dispose command +Info: Creating the `test`.`TestSchemaVersions` table +DB Operation: Execute non query command: CREATE TABLE `test`.`TestSchemaVersions` +( + ScriptName String, + Applied DateTime +) +ENGINE = MergeTree() +ORDER BY (ScriptName) +DB Operation: Dispose command +Info: The `test`.`TestSchemaVersions` table has been created +DB Operation: Execute non query command: script1contents +DB Operation: Dispose command +DB Operation: Create parameter +Info: DB Operation: Add parameter to command: scriptName=Script0001.sql +DB Operation: Create parameter +Info: DB Operation: Add parameter to command: applied= +DB Operation: Execute non query command: INSERT INTO `test`.`TestSchemaVersions` (ScriptName, Applied) VALUES (@scriptName, @applied) +DB Operation: Dispose command +Info: Upgrade successful +DB Operation: Dispose connection diff --git a/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyVariableSubstitutions.approved.txt b/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyVariableSubstitutions.approved.txt new file mode 100644 index 0000000..05ae6d6 --- /dev/null +++ b/src/Tests/ApprovalFiles/DatabaseSupportTests.VerifyVariableSubstitutions.approved.txt @@ -0,0 +1,30 @@ +DB Operation: Open connection +Info: Beginning database upgrade +Info: Checking whether journal table exists +DB Operation: Execute scalar command: EXISTS TABLE schemaversions +DB Operation: Dispose command +Info: Journal table does not exist +Info: Executing Database Server script 'Script0001.sql' +Info: Checking whether journal table exists +DB Operation: Execute scalar command: EXISTS TABLE schemaversions +DB Operation: Dispose command +Info: Creating the `schemaversions` table +DB Operation: Execute non query command: CREATE TABLE `schemaversions` +( + ScriptName String, + Applied DateTime +) +ENGINE = MergeTree() +ORDER BY (ScriptName) +DB Operation: Dispose command +Info: The `schemaversions` table has been created +DB Operation: Execute non query command: print SubstitutedValue +DB Operation: Dispose command +DB Operation: Create parameter +Info: DB Operation: Add parameter to command: scriptName=Script0001.sql +DB Operation: Create parameter +Info: DB Operation: Add parameter to command: applied= +DB Operation: Execute non query command: INSERT INTO `schemaversions` (ScriptName, Applied) VALUES (@scriptName, @applied) +DB Operation: Dispose command +Info: Upgrade successful +DB Operation: Dispose connection diff --git a/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs b/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs index edee20c..c7d2b61 100644 --- a/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs +++ b/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs @@ -1,15 +1,41 @@ namespace DbUp.ClickHouse { + public class ClickHouseConnectionManager : DbUp.Engine.Transactions.DatabaseConnectionManager, DbUp.Engine.Transactions.IConnectionManager + { + public ClickHouseConnectionManager(string connectionString) { } + public override System.Collections.Generic.IEnumerable SplitScriptIntoCommands(string scriptContents) { } + } public static class ClickHouseExtensions { - public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supportedDatabases, string connectionString) { } + public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(DbUp.Engine.Transactions.IConnectionManager connectionManager) { } + public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supported, string connectionString) { } + public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supported, DbUp.Engine.Transactions.IConnectionManager connectionManager) { } + public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(DbUp.Engine.Transactions.IConnectionManager connectionManager, string schema) { } + public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supported, string connectionString, string database) { } + public static DbUp.Builder.UpgradeEngineBuilder JournalToClickHouseTable(this DbUp.Builder.UpgradeEngineBuilder builder, string schema, string table) { } + } + public class ClickHouseJournal : DbUp.Support.TableJournal, DbUp.Engine.IJournal + { + public ClickHouseJournal(System.Func connectionManager, System.Func logger, string schema, string tableName) { } + protected override string CreateSchemaTableSql(string quotedPrimaryKeyName) { } + protected override string DoesTableExistSql() { } + protected override string GetInsertJournalEntrySql(string scriptName, string applied) { } + protected override string GetJournalEntriesSql() { } + } + public class ClickHouseObjectParser : DbUp.Support.SqlObjectParser, DbUp.Engine.ISqlObjectParser + { + public ClickHouseObjectParser() { } + } + public class ClickHousePreprocessor : DbUp.Engine.IScriptPreprocessor + { + public ClickHousePreprocessor() { } + public string Process(string contents) { } } - public class ClickHouseJournal : DbUp.Engine.IJournal + public class ClickHouseScriptExecutor : DbUp.Support.ScriptExecutor, DbUp.Engine.IScriptExecutor { - public ClickHouseJournal(System.Func connectionManagerFactory, System.Func logFactory, string tableName) { } - public void EnsureTableExistsAndIsLatestVersion(System.Func dbCommandFactory) { } - public string[] GetExecutedScripts() { } - public void StoreExecutedScript(DbUp.Engine.SqlScript script, System.Func dbCommandFactory) { } + public ClickHouseScriptExecutor(System.Func connectionManagerFactory, System.Func log, string schema, System.Func variablesEnabled, System.Collections.Generic.IEnumerable scriptPreprocessors, System.Func journalFactory) { } + protected override void ExecuteCommandsWithinExceptionHandler(int index, DbUp.Engine.SqlScript script, System.Action executeCommand) { } + protected override string GetVerifySchemaSql(string schema) { } } } diff --git a/src/Tests/ClickHouseContainerFixture.cs b/src/Tests/ClickHouseContainerFixture.cs new file mode 100644 index 0000000..f7d6f20 --- /dev/null +++ b/src/Tests/ClickHouseContainerFixture.cs @@ -0,0 +1,34 @@ +using Testcontainers.ClickHouse; +using Xunit; + +namespace DbUp.ClickHouse.Tests; + +/// +/// Class fixture for a ClickHouse container that is shared across all tests in a test class. +/// This ensures the container is created once per test class rather than once per test. +/// +public class ClickHouseContainerFixture : IAsyncLifetime +{ + private ClickHouseContainer? clickhouseContainer; + public string ConnectionString { get; private set; } = string.Empty; + + public async Task InitializeAsync() + { + // Create and start a ClickHouse container + var container = new ClickHouseBuilder().WithDatabase("testdb") + .Build(); + + await container.StartAsync(); + + ConnectionString = container.GetConnectionString(); + clickhouseContainer = container; + } + + public async Task DisposeAsync() + { + if (clickhouseContainer != null) + { + await clickhouseContainer.DisposeAsync(); + } + } +} diff --git a/src/Tests/ClickHouseIntegrationTests.cs b/src/Tests/ClickHouseIntegrationTests.cs new file mode 100644 index 0000000..b98ef64 --- /dev/null +++ b/src/Tests/ClickHouseIntegrationTests.cs @@ -0,0 +1,435 @@ +using ClickHouse.Driver.ADO; +using Shouldly; +using Xunit; +using Xunit.Abstractions; + +namespace DbUp.ClickHouse.Tests; + +/// +/// Integration tests for ClickHouse database operations using TestContainers. +/// Tests both script execution and journal table operations against a real ClickHouse instance. +/// Uses class fixture to share container across all tests in the class. +/// +public class ClickHouseIntegrationTests : IClassFixture, IDisposable +{ + private readonly ClickHouseContainerFixture fixture; + readonly ITestOutputHelper testOutputHelper; + string ConnectionString => fixture.ConnectionString; + + public ClickHouseIntegrationTests(ClickHouseContainerFixture fixture, ITestOutputHelper testOutputHelper) + { + this.fixture = fixture; + this.testOutputHelper = testOutputHelper; + } + + public void Dispose() + { + // Clean up after each test to ensure test isolation + try + { + using var connection = new ClickHouseConnection(ConnectionString); + connection.Open(); + using var command = connection.CreateCommand(); + + // Clear the journal table to ensure clean state for next test + command.CommandText = "DROP TABLE IF EXISTS `testdb`.`schemaversions`"; + command.ExecuteNonQuery(); + + // Also clean up any test tables that might have been created + command.CommandText = "DROP TABLE IF EXISTS test_table_ddl"; + command.ExecuteNonQuery(); + + command.CommandText = "DROP TABLE IF EXISTS insert_test_table"; + command.ExecuteNonQuery(); + + command.CommandText = "DROP TABLE IF EXISTS multi_statement_test"; + command.ExecuteNonQuery(); + + command.CommandText = "DROP TABLE IF EXISTS duplicate_test"; + command.ExecuteNonQuery(); + + // Clean up custom database + command.CommandText = "DROP DATABASE IF EXISTS custom_db"; + command.ExecuteNonQuery(); + } + catch + { + // Ignore cleanup errors - they shouldn't fail the tests + } + } + [Fact] + public void ScriptExecution_DDL_CreateTable_ShouldExecuteSuccessfully() + { + // Arrange + var upgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_CreateTestTable.sql", @" + CREATE TABLE test_table_ddl + ( + id UInt32, + name String, + created_at DateTime + ) + ENGINE = MergeTree() + ORDER BY id") + .LogToConsole() + .Build(); + + // Act + var result = upgrader.PerformUpgrade(); + + // Assert + if (!result.Successful) + { + testOutputHelper.WriteLine($"[DEBUG_LOG] DbUp failed with error: {result.Error}"); + throw new Exception($"DbUp operation failed: {result.Error}"); + } + result.Successful.ShouldBeTrue(); + result.Error.ShouldBeNull(); + var scriptsList = result.Scripts.ToList(); + scriptsList.Count.ShouldBe(1); + scriptsList[0].Name.ShouldBe("001_CreateTestTable.sql"); + + // Verify table was created + using var connection = new ClickHouseConnection(ConnectionString); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "EXISTS TABLE test_table_ddl"; + var exists = command.ExecuteScalar(); + exists.ShouldBe(1); + } + + [Fact] + public void ScriptExecution_DML_InsertData_ShouldExecuteSuccessfully() + { + // Arrange - First create a table (not tracked in journal) + using (var setupConnection = new ClickHouseConnection(ConnectionString)) + { + setupConnection.Open(); + using var setupCommand = setupConnection.CreateCommand(); + setupCommand.CommandText = @" + CREATE TABLE IF NOT EXISTS insert_test_table + ( + id UInt32, + name String + ) + ENGINE = MergeTree() + ORDER BY id"; + setupCommand.ExecuteNonQuery(); + } + + var upgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_InsertData.sql", @" + INSERT INTO insert_test_table (id, name) VALUES + (1, 'Test Record 1'), + (2, 'Test Record 2'), + (3, 'Test Record 3')") + .LogToConsole() + .Build(); + + // Act + var result = upgrader.PerformUpgrade(); + + // Assert + result.Successful.ShouldBeTrue(); + result.Error.ShouldBeNull(); + var scriptsList = result.Scripts.ToList(); + scriptsList.Count.ShouldBe(1); + + // Verify data was inserted + using var connection = new ClickHouseConnection(ConnectionString); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT COUNT(*) FROM insert_test_table"; + var count = command.ExecuteScalar(); + count.ShouldBe(3); + } + + [Fact] + public void ScriptExecution_MultipleStatements_ShouldExecuteSuccessfully() + { + // Arrange + var upgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_MultipleStatements.sql", @" + CREATE TABLE multi_statement_test1 + ( + id UInt32, + name String + ) + ENGINE = MergeTree() + ORDER BY id; + + CREATE TABLE multi_statement_test2 + ( + id UInt32, + description String + ) + ENGINE = MergeTree() + ORDER BY id; + + INSERT INTO multi_statement_test1 (id, name) VALUES (1, 'Test')") + .LogToConsole() + .Build(); + + // Act + var result = upgrader.PerformUpgrade(); + + // Assert + result.Successful.ShouldBeTrue(); + result.Error.ShouldBeNull(); + + // Verify both tables were created and data was inserted + using var connection = new ClickHouseConnection(ConnectionString); + connection.Open(); + using var command1 = connection.CreateCommand(); + command1.CommandText = "EXISTS TABLE multi_statement_test1"; + command1.ExecuteScalar().ShouldBe(1); + + using var command2 = connection.CreateCommand(); + command2.CommandText = "EXISTS TABLE multi_statement_test2"; + command2.ExecuteScalar().ShouldBe(1); + + using var command3 = connection.CreateCommand(); + command3.CommandText = "SELECT COUNT(*) FROM multi_statement_test1"; + command3.ExecuteScalar().ShouldBe(1); + } + + [Fact] + public void ScriptExecution_InvalidSQL_ShouldFailGracefully() + { + // Arrange + var upgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_InvalidSQL.sql", "INVALID SQL STATEMENT THAT SHOULD FAIL") + .LogToConsole() + .Build(); + + // Act + var result = upgrader.PerformUpgrade(); + + // Assert + result.Successful.ShouldBeFalse(); + result.Error.ShouldNotBeNull(); + } + + [Fact] + public void JournalTable_Creation_ShouldCreateCorrectSchema() + { + // Arrange & Act - The journal table should be created automatically + var upgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_TestScript.sql", "SELECT 1") + .LogToConsole() + .Build(); + + var result = upgrader.PerformUpgrade(); + + // Assert + result.ShouldSatisfyAllConditions( + x => x.Successful.ShouldBeTrue(), + x => x.Error.ShouldBeNull(), + x => x.Scripts.Count().ShouldBe(1), + x => x.Scripts.First().Name.ShouldBe("001_TestScript.sql") + ); + + // Verify the journal table exists and has the correct structure + using var connection = new ClickHouseConnection(ConnectionString); + connection.Open(); + using var command = connection.CreateCommand(); + + // Check if the journal table exists and get its structure + command.CommandText = "DESCRIBE TABLE `testdb`.`schemaversions`"; + using var reader = command.ExecuteReader(); + + var columns = new List<(string Name, string Type)>(); + while (reader.Read()) + { + columns.Add((reader["name"].ToString() ?? "", reader["type"].ToString() ?? "")); + } + + // Verify the schema + columns.Count.ShouldBe(2); + columns.ShouldContain(c => c.Name == "ScriptName" && c.Type == "String"); + columns.ShouldContain(c => c.Name == "Applied" && c.Type == "DateTime"); + } + + [Fact] + public void JournalTable_StoreExecutedScript_ShouldRecordExecution() + { + // Arrange & Act + var upgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_FirstScript.sql", "SELECT 1 as test_column") + .WithScript("002_SecondScript.sql", "SELECT 2 as test_column") + .LogToConsole() + .Build(); + + var result = upgrader.PerformUpgrade(); + + // Assert + result.Successful.ShouldBeTrue(); + + // Verify scripts were recorded in journal + using var connection = new ClickHouseConnection(ConnectionString); + connection.Open(); + using var command = connection.CreateCommand(); + + command.CommandText = "SELECT ScriptName FROM `testdb`.`schemaversions` ORDER BY ScriptName"; + using var reader = command.ExecuteReader(); + + var scripts = new List(); + while (reader.Read()) + { + scripts.Add(reader["ScriptName"].ToString() ?? ""); + } + + scripts.Count.ShouldBe(2); + scripts.ShouldContain("001_FirstScript.sql"); + scripts.ShouldContain("002_SecondScript.sql"); + } + + [Fact] + public void JournalTable_RetrieveExecutedScripts_ShouldReturnCorrectList() + { + // Arrange - Execute some scripts first + var firstUpgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_InitialScript.sql", "SELECT 1") + .WithScript("002_SecondScript.sql", "SELECT 2") + .LogToConsole() + .Build(); + + firstUpgrader.PerformUpgrade(); + + // Act - Try to run upgrader again with additional scripts + var secondUpgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_InitialScript.sql", "SELECT 1") // Should be skipped + .WithScript("002_SecondScript.sql", "SELECT 2") // Should be skipped + .WithScript("003_NewScript.sql", "SELECT 3") // Should be executed + .LogToConsole() + .Build(); + + var result = secondUpgrader.PerformUpgrade(); + + // Assert + result.Successful.ShouldBeTrue(); + // Only the new script should have been executed + var scriptsList = result.Scripts.ToList(); + scriptsList.Count.ShouldBe(1); + scriptsList[0].Name.ShouldBe("003_NewScript.sql"); + } + + [Fact] + public void JournalTable_DuplicateScriptDetection_ShouldSkipAlreadyExecuted() + { + // Arrange - Execute a script + var firstUpgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_TestScript.sql", "CREATE TABLE duplicate_test (id UInt32) ENGINE = MergeTree() ORDER BY id") + .LogToConsole() + .Build(); + + var firstResult = firstUpgrader.PerformUpgrade(); + firstResult.Successful.ShouldBeTrue(); + + // Act - Try to run the same script again + var secondUpgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString) + .WithScript("001_TestScript.sql", "CREATE TABLE duplicate_test (id UInt32) ENGINE = MergeTree() ORDER BY id") + .LogToConsole() + .Build(); + + var secondResult = secondUpgrader.PerformUpgrade(); + + // Assert + secondResult.Successful.ShouldBeTrue(); + // No scripts should have been executed the second time + var scriptsList = secondResult.Scripts.ToList(); + scriptsList.Count.ShouldBe(0); + } + + [Fact] + public void JournalTable_CustomDatabase_ShouldWorkCorrectly() + { + // Arrange - Use a custom database + var customConnectionString = ConnectionString.Replace("Database=testdb", "Database=custom_db"); + + // First create the custom database + using (var connection = new ClickHouseConnection(ConnectionString)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "CREATE DATABASE IF NOT EXISTS custom_db"; + command.ExecuteNonQuery(); + } + + var upgrader = DeployChanges.To + .ClickHouseDatabase(customConnectionString) + .WithScript("001_CustomDatabaseTest.sql", "SELECT 1") + .LogToConsole() + .Build(); + + // Act + var result = upgrader.PerformUpgrade(); + + // Assert + result.Successful.ShouldBeTrue(); + + // Verify journal table was created in a custom database + using var customConnection = new ClickHouseConnection(customConnectionString); + customConnection.Open(); + using var verifyCommand = customConnection.CreateCommand(); + verifyCommand.CommandText = "SELECT ScriptName FROM `custom_db`.`schemaversions`"; + using var reader = verifyCommand.ExecuteReader(); + + reader.Read().ShouldBeTrue(); + (reader["ScriptName"].ToString() ?? "").ShouldBe("001_CustomDatabaseTest.sql"); + } + + [Fact] + public void JournalTable_CustomSchema_ShouldCreateInSpecifiedSchema() + { + // Arrange - Use a custom schema within the default database + const string customSchema = "custom_schema"; + + // First, create the custom schema (database in ClickHouse) + using (var setupConnection = new ClickHouseConnection(ConnectionString)) + { + setupConnection.Open(); + using var setupCommand = setupConnection.CreateCommand(); + setupCommand.CommandText = $"CREATE DATABASE IF NOT EXISTS {customSchema}"; + setupCommand.ExecuteNonQuery(); + } + + var upgrader = DeployChanges.To + .ClickHouseDatabase(ConnectionString, customSchema) + .WithScript("001_CustomSchemaTest.sql", "SELECT 1") + .LogToConsole() + .Build(); + + // Act + var result = upgrader.PerformUpgrade(); + + // Assert + result.Successful.ShouldBeTrue(); + + // Verify journal table was created in the custom schema + using var verifyConnection = new ClickHouseConnection(ConnectionString); + verifyConnection.Open(); + using var verifyCommand = verifyConnection.CreateCommand(); + verifyCommand.CommandText = $"SELECT ScriptName FROM `{customSchema}`.`schemaversions`"; + using var reader = verifyCommand.ExecuteReader(); + + reader.Read().ShouldBeTrue(); + (reader["ScriptName"].ToString() ?? "").ShouldBe("001_CustomSchemaTest.sql"); + + // Verify the table exists in the correct schema + using var schemaCheckCommand = verifyConnection.CreateCommand(); + schemaCheckCommand.CommandText = $"SELECT 1 FROM information_schema.tables WHERE table_name = 'schemaversions' AND table_schema = '{customSchema}'"; + var tableExists = schemaCheckCommand.ExecuteScalar(); + tableExists.ShouldNotBeNull(); + } +} diff --git a/src/Tests/ClickHouseJournalTests.cs b/src/Tests/ClickHouseJournalTests.cs new file mode 100644 index 0000000..13d7d64 --- /dev/null +++ b/src/Tests/ClickHouseJournalTests.cs @@ -0,0 +1,111 @@ +using System.Data; +using DbUp.Engine; +using DbUp.Engine.Output; +using DbUp.Engine.Transactions; +using DbUp.Tests.Common; +using NSubstitute; +using Shouldly; +using Xunit; + +namespace DbUp.ClickHouse.Tests; + +public class ClickHouseJournalTests +{ + // Test helper class to expose protected methods + private class TestableClickHouseJournal : ClickHouseJournal + { + public TestableClickHouseJournal( + Func connectionManager, + Func logger, + string schema, + string tableName) + : base(connectionManager, logger, schema, tableName) + { + } + + public string TestGetJournalEntriesSql() => GetJournalEntriesSql(); + public string TestGetInsertJournalEntrySql(string scriptName, string applied) => GetInsertJournalEntrySql(scriptName, applied); + } + + [Fact] + public void StoreExecutedScript_CreatesCorrectInsertStatement() + { + // Arrange + var dbConnection = Substitute.For(); + var connectionManager = new TestConnectionManager(dbConnection); + var command = Substitute.For(); + var param1 = Substitute.For(); + var param2 = Substitute.For(); + dbConnection.CreateCommand().Returns(command); + command.CreateParameter().Returns(param1, param2); + command.ExecuteScalar().Returns(x => 0); + var consoleUpgradeLog = new ConsoleUpgradeLog(); + var journal = new ClickHouseJournal(() => connectionManager, () => consoleUpgradeLog, "default", + "SchemaVersions"); + + // Act + journal.StoreExecutedScript(new SqlScript("test", "select 1"), () => command); + + // Assert + command.Received(2).CreateParameter(); + command.CommandText.ShouldBe("INSERT INTO `default`.`SchemaVersions` (ScriptName, Applied) VALUES (@scriptName, @applied)"); + command.Received().ExecuteNonQuery(); + } + + [Fact] + public void GetJournalEntriesSql_GeneratesCorrectSelectStatement() + { + // Arrange + var dbConnection = Substitute.For(); + var connectionManager = new TestConnectionManager(dbConnection); + var consoleUpgradeLog = new ConsoleUpgradeLog(); + var journal = new TestableClickHouseJournal(() => connectionManager, () => consoleUpgradeLog, "default", + "SchemaVersions"); + + // Act + var sql = journal.TestGetJournalEntriesSql(); + + // Assert + sql.ShouldBe("SELECT ScriptName FROM `default`.`SchemaVersions` ORDER BY ScriptName"); + } + + [Fact] + public void GetInsertJournalEntrySql_GeneratesCorrectInsertStatement() + { + // Arrange + var dbConnection = Substitute.For(); + var connectionManager = new TestConnectionManager(dbConnection); + var consoleUpgradeLog = new ConsoleUpgradeLog(); + var journal = new TestableClickHouseJournal(() => connectionManager, () => consoleUpgradeLog, "default", + "SchemaVersions"); + + // Act + var sql = journal.TestGetInsertJournalEntrySql("@scriptName", "@applied"); + + // Assert + sql.ShouldBe("INSERT INTO `default`.`SchemaVersions` (ScriptName, Applied) VALUES (@scriptName, @applied)"); + } + + [Fact] + public void EnsureTableExists_CreatesCorrectTableCreationSql() + { + // Arrange + var dbConnection = Substitute.For(); + var connectionManager = new TestConnectionManager(dbConnection); + var command = Substitute.For(); + dbConnection.CreateCommand().Returns(command); + command.ExecuteScalar().Returns(0); // Table doesn't exist + var consoleUpgradeLog = new ConsoleUpgradeLog(); + var journal = new ClickHouseJournal(() => connectionManager, () => consoleUpgradeLog, "default", + "SchemaVersions"); + + // Act + journal.EnsureTableExistsAndIsLatestVersion(() => command); + + // Assert + // Verify the create table command was executed + command.Received().ExecuteNonQuery(); + // The exact SQL will be set when the table doesn't exist + command.CommandText.ShouldNotBeNullOrEmpty(); + } +} diff --git a/src/Tests/ClickHouseQueryParserTests.cs b/src/Tests/ClickHouseQueryParserTests.cs new file mode 100644 index 0000000..2382e43 --- /dev/null +++ b/src/Tests/ClickHouseQueryParserTests.cs @@ -0,0 +1,90 @@ +using Xunit; + +namespace DbUp.ClickHouse.Tests; + +public class ClickHouseQueryParserTests +{ + [Theory] + [InlineData("SELECT 1\n;\nSELECT 2", 2, "SELECT 1", "SELECT 2")] + [InlineData(";;SELECT 1", 1, "SELECT 1")] + [InlineData("SELECT 1;", 1, "SELECT 1")] + [InlineData("", 0)] + // ClickHouse parser now properly handles comments - doesn't split on semicolon inside comments + [InlineData("SELECT 1 /* block comment; */", 1, "SELECT 1 /* block comment; */")] + [InlineData( + """ + SELECT 1; + -- Line comment; with semicolon + SELECT 2; + """, 2, + "SELECT 1", + "-- Line comment; with semicolon\r\nSELECT 2")] + // ClickHouse parser now properly handles string literals - doesn't split on semicolon inside strings + [InlineData("SELECT 'string with; semicolon'", 1, "SELECT 'string with; semicolon'")] + // ClickHouse parser now properly handles quoted identifiers - doesn't split on semicolon inside quotes + [InlineData("SELECT 1 as `QUOTED;IDENT`", 1, "SELECT 1 as `QUOTED;IDENT`")] + [InlineData(""" + CREATE TABLE test ( + id UInt32, + name String + ) ENGINE = MergeTree() + ORDER BY id; + INSERT INTO test VALUES (1, 'test'); + """, 2)] + [InlineData(""" + CREATE VIEW test_view AS + SELECT * FROM test; + SELECT COUNT(*) FROM test_view; + """, 2)] + public void split_into_statements(string sql, int statementCount, params string[] expected) + { + var results = ParseCommand(sql); + Assert.Equal(statementCount, results.Count); + if (expected.Length > 0) + Assert.Equal(expected, results); + } + + [Fact] + public void split_single_statement_no_semicolon() + { + const string sql = "SELECT 1"; + var results = ParseCommand(sql); + Assert.Single(results); + Assert.Equal(sql, results[0]); + } + + [Fact] + public void split_handles_empty_statements() + { + const string sql = ";;; SELECT 1 ;;;"; + var results = ParseCommand(sql); + Assert.Single(results); + Assert.Equal("SELECT 1", results[0]); + } + + [Fact] + public void split_handles_multiline_statements() + { + const string sql = """ + CREATE TABLE test ( + id UInt32, + name String + ) ENGINE = MergeTree() + ORDER BY id; + + INSERT INTO test + VALUES (1, 'test') + """; + var results = ParseCommand(sql); + Assert.Equal(2, results.Count); + Assert.Contains("CREATE TABLE test", results[0]); + Assert.Contains("INSERT INTO test", results[1]); + } + + private static List ParseCommand(string sql) + { + var manager = new ClickHouseConnectionManager(""); + var commands = manager.SplitScriptIntoCommands(sql); + return commands.ToList(); + } +} diff --git a/src/Tests/DatabaseSupportTests.cs b/src/Tests/DatabaseSupportTests.cs index 100864a..23ee8b6 100644 --- a/src/Tests/DatabaseSupportTests.cs +++ b/src/Tests/DatabaseSupportTests.cs @@ -1,4 +1,4 @@ -using DbUp.Builder; +using DbUp.Builder; using DbUp.Tests.Common; namespace DbUp.ClickHouse.Tests; @@ -15,6 +15,6 @@ protected override UpgradeEngineBuilder DeployTo(SupportedDatabases to) protected override UpgradeEngineBuilder AddCustomNamedJournalToBuilder(UpgradeEngineBuilder builder, string schema, string tableName) => builder.JournalTo( (connectionManagerFactory, logFactory) - => new ClickHouseJournal(connectionManagerFactory, logFactory, tableName) + => new ClickHouseJournal(connectionManagerFactory, logFactory, schema, tableName) ); } diff --git a/src/Tests/NoPublicApiChanges.cs b/src/Tests/NoPublicApiChanges.cs index b98c14d..ee94d67 100644 --- a/src/Tests/NoPublicApiChanges.cs +++ b/src/Tests/NoPublicApiChanges.cs @@ -1,4 +1,4 @@ -using DbUp.Tests.Common; +using DbUp.Tests.Common; namespace DbUp.ClickHouse.Tests; diff --git a/src/Tests/Tests.csproj b/src/Tests/Tests.csproj index d036074..13c29eb 100644 --- a/src/Tests/Tests.csproj +++ b/src/Tests/Tests.csproj @@ -9,19 +9,22 @@ - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive + - - + + diff --git a/src/dbup-clickhouse.sln b/src/dbup-clickhouse.sln index f9793b1..49a274f 100644 --- a/src/dbup-clickhouse.sln +++ b/src/dbup-clickhouse.sln @@ -22,7 +22,11 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{77157734-01DA-4AA3-A15C-504013343B29}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig - dbup-clickhouse.sln.DotSettings = dbup-clickhouse.sln.DotSettings + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".Solution Items", ".Solution Items", "{CC2BF5D2-DA01-4524-BA87-488F9C164356}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig Directory.Build.props = Directory.Build.props EndProjectSection EndProject diff --git a/src/dbup-clickhouse/ClickHouseConnectionManager.cs b/src/dbup-clickhouse/ClickHouseConnectionManager.cs new file mode 100644 index 0000000..9b396c8 --- /dev/null +++ b/src/dbup-clickhouse/ClickHouseConnectionManager.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using System.Data; +using System.Linq; +using ClickHouse.Driver.ADO; +using DbUp.Engine.Transactions; + +namespace DbUp.ClickHouse; + +/// +/// Manages ClickHouse database connections. +/// +public class ClickHouseConnectionManager : DatabaseConnectionManager +{ + /// + /// Creates a new ClickHouse database connection. + /// + /// The ClickHouse connection string. + public ClickHouseConnectionManager(string connectionString) + : base(new DelegateConnectionFactory(_ => CreateConnection(connectionString))) + { + } + + private static IDbConnection CreateConnection(string connectionString) + { + return new ClickHouseConnection(connectionString); + } + + /// + /// Splits the statements in the script using proper SQL parsing that handles comments and string literals. + /// + /// The contents of the script to split. + public override IEnumerable SplitScriptIntoCommands(string scriptContents) + { + var statements = ClickHouseQueryParser.ParseRawQuery(scriptContents); + return statements + .Select(s => s.Trim()) + .Where(s => s.Length > 0) + .ToArray(); + } +} + diff --git a/src/dbup-clickhouse/ClickHouseExtensions.cs b/src/dbup-clickhouse/ClickHouseExtensions.cs index 17823ee..f644a54 100644 --- a/src/dbup-clickhouse/ClickHouseExtensions.cs +++ b/src/dbup-clickhouse/ClickHouseExtensions.cs @@ -1,12 +1,39 @@ -using System; using DbUp.Builder; +using DbUp.Engine.Transactions; namespace DbUp.ClickHouse; +/// +/// Configuration extension methods for ClickHouse. +/// public static class ClickHouseExtensions { - public static UpgradeEngineBuilder ClickHouseDatabase(this SupportedDatabases supportedDatabases, string connectionString) + public static UpgradeEngineBuilder ClickHouseDatabase(this SupportedDatabases supported, string connectionString) + => ClickHouseDatabase(supported, connectionString, null); + + public static UpgradeEngineBuilder ClickHouseDatabase(this SupportedDatabases supported, string connectionString, string database) + => ClickHouseDatabase(new ClickHouseConnectionManager(connectionString), database); + + public static UpgradeEngineBuilder ClickHouseDatabase(this SupportedDatabases supported, IConnectionManager connectionManager) + => ClickHouseDatabase(connectionManager); + + public static UpgradeEngineBuilder ClickHouseDatabase(IConnectionManager connectionManager) + => ClickHouseDatabase(connectionManager, null); + + public static UpgradeEngineBuilder ClickHouseDatabase(IConnectionManager connectionManager, string schema) { - throw new NotImplementedException(); + var builder = new UpgradeEngineBuilder(); + builder.Configure(c => c.ConnectionManager = connectionManager); + builder.Configure(c => c.ScriptExecutor = new ClickHouseScriptExecutor(() => c.ConnectionManager, () => c.Log, schema, () => c.VariablesEnabled, c.ScriptPreprocessors, () => c.Journal)); + builder.Configure(c => c.Journal = new ClickHouseJournal(() => c.ConnectionManager, () => c.Log, schema, "schemaversions")); + builder.WithPreprocessor(new ClickHousePreprocessor()); + return builder; + } + + public static UpgradeEngineBuilder JournalToClickHouseTable(this UpgradeEngineBuilder builder, string schema, string table) + { + builder.Configure(c => c.Journal = new ClickHouseJournal(() => c.ConnectionManager, () => c.Log, schema, table)); + return builder; } } + diff --git a/src/dbup-clickhouse/ClickHouseJournal.cs b/src/dbup-clickhouse/ClickHouseJournal.cs index f5b504a..2bfe9ee 100644 --- a/src/dbup-clickhouse/ClickHouseJournal.cs +++ b/src/dbup-clickhouse/ClickHouseJournal.cs @@ -1,23 +1,44 @@ -using System; -using System.Data; -using DbUp.Engine; using DbUp.Engine.Output; using DbUp.Engine.Transactions; +using DbUp.Support; namespace DbUp.ClickHouse; -public class ClickHouseJournal( -// Remove pragma once implemented -#pragma warning disable CS9113 // Parameter is unread. - Func connectionManagerFactory, - Func logFactory, - string tableName -#pragma warning restore CS9113 // Parameter is unread. - ) : IJournal +/// +/// Tracks the list of executed scripts in a ClickHouse table. +/// +public class ClickHouseJournal : TableJournal { - public string[] GetExecutedScripts() => throw new NotImplementedException(); + public ClickHouseJournal( + System.Func connectionManager, + System.Func logger, + string schema, + string tableName) + : base(connectionManager, logger, new ClickHouseObjectParser(), schema, tableName) + { + } - public void StoreExecutedScript(SqlScript script, Func dbCommandFactory) => throw new NotImplementedException(); + protected override string GetInsertJournalEntrySql(string scriptName, string applied) + => $"INSERT INTO {FqSchemaTableName} (ScriptName, Applied) VALUES ({scriptName}, {applied})"; - public void EnsureTableExistsAndIsLatestVersion(Func dbCommandFactory) => throw new NotImplementedException(); + protected override string GetJournalEntriesSql() + => $"SELECT ScriptName FROM {FqSchemaTableName} ORDER BY ScriptName"; + + protected override string CreateSchemaTableSql(string quotedPrimaryKeyName) + => $""" + CREATE TABLE {FqSchemaTableName} + ( + ScriptName String, + Applied DateTime + ) + ENGINE = MergeTree() + ORDER BY (ScriptName) + """; + protected override string DoesTableExistSql() + { + return string.IsNullOrEmpty(SchemaTableSchema) + ? $"EXISTS TABLE {UnquotedSchemaTableName}" + : $"EXISTS TABLE `{SchemaTableSchema}`.`{UnquotedSchemaTableName}`"; + } } + diff --git a/src/dbup-clickhouse/ClickHouseObjectParser.cs b/src/dbup-clickhouse/ClickHouseObjectParser.cs new file mode 100644 index 0000000..5c7e1fc --- /dev/null +++ b/src/dbup-clickhouse/ClickHouseObjectParser.cs @@ -0,0 +1,9 @@ +using DbUp.Support; + +namespace DbUp.ClickHouse; + +/// +/// Parses SQL Objects and performs quoting functions for ClickHouse. +/// +public class ClickHouseObjectParser() : SqlObjectParser("`", "`"); + diff --git a/src/dbup-clickhouse/ClickHousePreprocessor.cs b/src/dbup-clickhouse/ClickHousePreprocessor.cs new file mode 100644 index 0000000..6b06791 --- /dev/null +++ b/src/dbup-clickhouse/ClickHousePreprocessor.cs @@ -0,0 +1,12 @@ +using DbUp.Engine; + +namespace DbUp.ClickHouse; + +/// +/// Pass-through preprocessor for ClickHouse scripts. +/// +public class ClickHousePreprocessor : IScriptPreprocessor +{ + public string Process(string contents) => contents; +} + diff --git a/src/dbup-clickhouse/ClickHouseQueryParser.cs b/src/dbup-clickhouse/ClickHouseQueryParser.cs new file mode 100644 index 0000000..1761e14 --- /dev/null +++ b/src/dbup-clickhouse/ClickHouseQueryParser.cs @@ -0,0 +1,177 @@ +#nullable enable +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; + +namespace DbUp.ClickHouse; + +internal static class ClickHouseQueryParser +{ + public static IReadOnlyCollection ParseRawQuery(string sql) + { + List result = new(); + StringBuilder currentStatementBuilder = new(); + + currentStatementBuilder.Clear(); + + var currCharOfs = 0; + var end = sql.Length; + var ch = '\0'; + var currTokenBeg = 0; + var blockCommentLevel = 0; + var parenthesisLevel = 0; + + None: + if (currCharOfs >= end) + goto Finish; + var lastChar = ch; + ch = sql[currCharOfs++]; + NoneContinue: + while (true) + { + switch (ch) + { + case '/': + goto BlockCommentBegin; + case '-': + goto LineCommentBegin; + case '\'': + goto Quoted; + case '`': + goto Quoted; + case ';': + if (parenthesisLevel == 0) + goto SemiColon; + break; + case '(': + parenthesisLevel++; + break; + case ')': + parenthesisLevel--; + break; + } + + if (currCharOfs >= end) + goto Finish; + + lastChar = ch; + ch = sql[currCharOfs++]; + } + + Quoted: + Debug.Assert(ch is '\'' or '`'); + while (currCharOfs < end && sql[currCharOfs] != ch) + { + currCharOfs++; + } + + if (currCharOfs < end) + { + currCharOfs++; + ch = '\0'; + goto None; + } + + goto Finish; + + LineCommentBegin: + if (currCharOfs < end) + { + ch = sql[currCharOfs++]; + if (ch == '-') + goto LineComment; + lastChar = '\0'; + goto NoneContinue; + } + + goto Finish; + + LineComment: + while (currCharOfs < end) + { + ch = sql[currCharOfs++]; + if (ch is '\r' or '\n') + goto None; + } + + goto Finish; + + BlockCommentBegin: + while (currCharOfs < end) + { + ch = sql[currCharOfs++]; + switch (ch) + { + case '*': + blockCommentLevel++; + goto BlockComment; + case '/': + continue; + } + + if (blockCommentLevel > 0) + goto BlockComment; + lastChar = '\0'; + goto NoneContinue; + } + + goto Finish; + + BlockComment: + while (currCharOfs < end) + { + ch = sql[currCharOfs++]; + switch (ch) + { + case '*': + goto BlockCommentEnd; + case '/': + goto BlockCommentBegin; + } + } + + goto Finish; + + BlockCommentEnd: + while (currCharOfs < end) + { + ch = sql[currCharOfs++]; + if (ch == '/') + { + if (--blockCommentLevel > 0) + goto BlockComment; + goto None; + } + + if (ch != '*') + goto BlockComment; + } + + goto Finish; + + SemiColon: + currentStatementBuilder.Append(sql, currTokenBeg, currCharOfs - currTokenBeg - 1); + result.Add(currentStatementBuilder.ToString()); + while (currCharOfs < end) + { + ch = sql[currCharOfs]; + if (char.IsWhiteSpace(ch)) + { + currCharOfs++; + continue; + } + + currentStatementBuilder.Clear(); + + currTokenBeg = currCharOfs; + goto None; + } + + return result; + + Finish: + currentStatementBuilder.Append(sql, currTokenBeg, end - currTokenBeg); + result.Add(currentStatementBuilder.ToString()); + return result; + } +} diff --git a/src/dbup-clickhouse/ClickHouseScriptExecutor.cs b/src/dbup-clickhouse/ClickHouseScriptExecutor.cs new file mode 100644 index 0000000..4f85685 --- /dev/null +++ b/src/dbup-clickhouse/ClickHouseScriptExecutor.cs @@ -0,0 +1,46 @@ +using System; +using DbUp.Engine; +using DbUp.Engine.Output; +using DbUp.Engine.Transactions; +using DbUp.Support; + +namespace DbUp.ClickHouse; + +/// +/// An implementation of that executes against a ClickHouse database. +/// +public class ClickHouseScriptExecutor : ScriptExecutor +{ + /// + /// Initializes an instance of the class. + /// + public ClickHouseScriptExecutor( + Func connectionManagerFactory, + Func log, + string schema, + Func variablesEnabled, + System.Collections.Generic.IEnumerable scriptPreprocessors, + Func journalFactory) + : base(connectionManagerFactory, new ClickHouseObjectParser(), log, schema, variablesEnabled, scriptPreprocessors, journalFactory) + { + } + + protected override string GetVerifySchemaSql(string schema) + => $"CREATE DATABASE IF NOT EXISTS {schema}"; + + protected override void ExecuteCommandsWithinExceptionHandler(int index, SqlScript script, Action executeCommand) + { + try + { + executeCommand(); + } + catch (Exception exception) + { + Log().LogInformation("ClickHouse exception has occurred in script: '{0}'", script.Name); + Log().LogError("Script block number: {0}; Message: {1}", index, exception.Message); + Log().LogError(exception.ToString()); + throw; + } + } +} + diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index f023af2..10c98f3 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -1,7 +1,7 @@  - DbUp makes it easy to deploy and upgrade SQL Server databases. This package adds ClickHouse support. + DbUp makes it easy to deploy and upgrade databases. This package adds ClickHouse support. DbUp ClickHouse Support DbUp Contributors DbUp @@ -17,6 +17,7 @@ + diff --git a/start-clickhouse.ps1 b/start-clickhouse.ps1 new file mode 100644 index 0000000..41126a3 --- /dev/null +++ b/start-clickhouse.ps1 @@ -0,0 +1,55 @@ +#!/usr/bin/env pwsh + +# Start ClickHouse 23.6-alpine Docker container +# Author: Generated script +# Description: Starts ClickHouse with username 'clickhouse' and password 'clickhouse' + +param( + [string]$ContainerName = "clickhouse-server", + [int]$Port = 8123, + [int]$NativePort = 9000 +) + +Write-Host "Starting ClickHouse 23.6-alpine Docker container..." -ForegroundColor Green + +# Stop and remove existing container if it exists +$existingContainer = docker ps -a -q -f name=$ContainerName +if ($existingContainer) { + Write-Host "Stopping existing container '$ContainerName'..." -ForegroundColor Yellow + docker stop $ContainerName | Out-Null + docker rm $ContainerName | Out-Null +} + +# Start new ClickHouse container +Write-Host "Starting new ClickHouse container..." -ForegroundColor Green + +docker run -d ` + --name $ContainerName ` + -p "${Port}:8123" ` + -p "${NativePort}:9000" ` + -e CLICKHOUSE_USER=clickhouse ` + -e CLICKHOUSE_PASSWORD=clickhouse ` + -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 ` + clickhouse/clickhouse-server:23.6-alpine + +if ($LASTEXITCODE -eq 0) { + Write-Host "ClickHouse container started successfully!" -ForegroundColor Green + Write-Host "" + Write-Host "Connection Details:" -ForegroundColor Cyan + Write-Host " HTTP Port: $Port" -ForegroundColor White + Write-Host " Native Port: $NativePort" -ForegroundColor White + Write-Host " Username: clickhouse" -ForegroundColor White + Write-Host " Password: clickhouse" -ForegroundColor White + Write-Host "" + Write-Host "HTTP URL: http://localhost:$Port" -ForegroundColor Yellow + Write-Host "Native URL: clickhouse://clickhouse:clickhouse@localhost:$NativePort" -ForegroundColor Yellow + Write-Host "" + Write-Host "To connect via clickhouse-client:" -ForegroundColor Cyan + Write-Host " docker exec -it $ContainerName clickhouse-client --user=clickhouse --password=clickhouse" -ForegroundColor White + Write-Host "" + Write-Host "To view logs:" -ForegroundColor Cyan + Write-Host " docker logs -f $ContainerName" -ForegroundColor White +} else { + Write-Host "Failed to start ClickHouse container!" -ForegroundColor Red + exit 1 +} From b7c2c6a25f5b5d00894a6a3105b59cff2ee419da Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Tue, 9 Sep 2025 16:54:41 +0500 Subject: [PATCH 02/17] Support multiple target frameworks and refactor SQL query parser --- src/dbup-clickhouse/ClickHouseQueryParser.cs | 316 +++++++++++-------- src/dbup-clickhouse/dbup-clickhouse.csproj | 4 +- 2 files changed, 184 insertions(+), 136 deletions(-) diff --git a/src/dbup-clickhouse/ClickHouseQueryParser.cs b/src/dbup-clickhouse/ClickHouseQueryParser.cs index 1761e14..7212b47 100644 --- a/src/dbup-clickhouse/ClickHouseQueryParser.cs +++ b/src/dbup-clickhouse/ClickHouseQueryParser.cs @@ -1,177 +1,227 @@ #nullable enable using System.Collections.Generic; -using System.Diagnostics; -using System.Text; namespace DbUp.ClickHouse; +/// +/// Provides SQL query parsing functionality for ClickHouse scripts, capable of splitting multi-statement SQL +/// into individual statements while properly handling comments, string literals, and quoted identifiers. +/// internal static class ClickHouseQueryParser { + /// + /// Represents the current parsing state when processing SQL text. + /// + private enum ParseState + { + /// Normal SQL parsing state. + Normal, + /// Inside a single-quoted string literal. + SingleQuote, + /// Inside a back-tick quoted identifier. + BackTickQuote, + /// Inside a line comment (-- style). + LineComment, + /// Inside a block comment (/* */ style). + BlockComment + } + + /// + /// Maintains parsing context and position information during SQL processing. + /// + private struct ParseContext() + { + /// Current character position in the SQL string. + public int Position { get; set; } = 0; + /// Starting position of the current SQL statement being parsed. + public int StatementStart { get; set; } = 0; + /// Current nesting level of parentheses to avoid splitting on semicolons within function calls or subqueries. + public int ParenthesisLevel { get; set; } = 0; + /// Current nesting level of block comments to handle nested /* */ comments correctly. + public int BlockCommentLevel { get; set; } = 0; + /// Current parsing state indicating the type of content being processed. + public ParseState State { get; set; } = ParseState.Normal; + } + + /// + /// Parses raw SQL text and splits it into individual executable statements. + /// + /// The SQL text to parse, which may contain multiple statements separated by semicolons. + /// + /// A read-only collection of individual SQL statements. Empty or whitespace-only statements are excluded. + /// Returns an empty collection if the input is null or empty. + /// + /// + /// This parser correctly handles: + /// - Semicolons within string literals ('text; with semicolon') + /// - Semicolons within quoted identifiers (`identifier; with semicolon`) + /// - Semicolons within line comments (-- comment; with semicolon) + /// - Semicolons within block comments (/* comment; with semicolon */) + /// - Nested block comments (/* outer /* inner */ outer */) + /// - Semicolons within parentheses for function calls and subqueries + /// public static IReadOnlyCollection ParseRawQuery(string sql) { - List result = new(); - StringBuilder currentStatementBuilder = new(); - - currentStatementBuilder.Clear(); - - var currCharOfs = 0; - var end = sql.Length; - var ch = '\0'; - var currTokenBeg = 0; - var blockCommentLevel = 0; - var parenthesisLevel = 0; - - None: - if (currCharOfs >= end) - goto Finish; - var lastChar = ch; - ch = sql[currCharOfs++]; - NoneContinue: - while (true) + if (string.IsNullOrEmpty(sql)) + return new List(); + + var statements = new List(); + var context = new ParseContext(); + + while (context.Position < sql.Length) { - switch (ch) + if (TryParseStatement(sql, context, out var statement)) { - case '/': - goto BlockCommentBegin; - case '-': - goto LineCommentBegin; - case '\'': - goto Quoted; - case '`': - goto Quoted; - case ';': - if (parenthesisLevel == 0) - goto SemiColon; - break; - case '(': - parenthesisLevel++; - break; - case ')': - parenthesisLevel--; - break; + statements.Add(statement); } - - if (currCharOfs >= end) - goto Finish; - - lastChar = ch; - ch = sql[currCharOfs++]; } - Quoted: - Debug.Assert(ch is '\'' or '`'); - while (currCharOfs < end && sql[currCharOfs] != ch) + // Add a final statement if any content remains + if (context.StatementStart < sql.Length) { - currCharOfs++; + var finalStatement = sql.Substring(context.StatementStart); + statements.Add(finalStatement); } - if (currCharOfs < end) + return statements; + } + + private static bool TryParseStatement(string sql, ParseContext context, out string statement) + { + statement = string.Empty; + + while (context.Position < sql.Length) { - currCharOfs++; - ch = '\0'; - goto None; + var ch = sql[context.Position]; + context.Position++; + + if (context.State == ParseState.Normal) + { + if (HandleNormalState(sql, context, ch, out statement)) + return true; + } + else + { + HandleSpecialStates(sql, context, ch); + } } - goto Finish; + return false; + } + + private static bool HandleNormalState(string sql, ParseContext context, char ch, out string statement) + { + statement = string.Empty; - LineCommentBegin: - if (currCharOfs < end) + switch (ch) { - ch = sql[currCharOfs++]; - if (ch == '-') - goto LineComment; - lastChar = '\0'; - goto NoneContinue; + case '\'': + context.State = ParseState.SingleQuote; + break; + case '`': + context.State = ParseState.BackTickQuote; + break; + case '/': + if (TryStartBlockComment(sql, context)) + context.State = ParseState.BlockComment; + break; + case '-': + if (TryStartLineComment(sql, context)) + context.State = ParseState.LineComment; + break; + case '(': + context.ParenthesisLevel++; + break; + case ')': + context.ParenthesisLevel--; + break; + case ';': + if (context.ParenthesisLevel == 0) + { + statement = ExtractStatement(sql, context); + SkipWhitespaceAndStartNext(sql, context); + return !string.IsNullOrEmpty(statement); + } + break; } - goto Finish; + return false; + } - LineComment: - while (currCharOfs < end) + private static void HandleSpecialStates(string sql, ParseContext context, char ch) + { + switch (context.State) { - ch = sql[currCharOfs++]; - if (ch is '\r' or '\n') - goto None; + case ParseState.SingleQuote: + if (ch == '\'') + context.State = ParseState.Normal; + break; + case ParseState.BackTickQuote: + if (ch == '`') + context.State = ParseState.Normal; + break; + case ParseState.LineComment: + if (ch is '\r' or '\n') + context.State = ParseState.Normal; + break; + case ParseState.BlockComment: + HandleBlockComment(sql, context, ch); + break; } + } - goto Finish; - - BlockCommentBegin: - while (currCharOfs < end) + private static bool TryStartBlockComment(string sql, ParseContext context) + { + if (context.Position < sql.Length && sql[context.Position] == '*') { - ch = sql[currCharOfs++]; - switch (ch) - { - case '*': - blockCommentLevel++; - goto BlockComment; - case '/': - continue; - } - - if (blockCommentLevel > 0) - goto BlockComment; - lastChar = '\0'; - goto NoneContinue; + context.Position++; + context.BlockCommentLevel = 1; + return true; } + return false; + } - goto Finish; - - BlockComment: - while (currCharOfs < end) + private static bool TryStartLineComment(string sql, ParseContext context) + { + if (context.Position < sql.Length && sql[context.Position] == '-') { - ch = sql[currCharOfs++]; - switch (ch) - { - case '*': - goto BlockCommentEnd; - case '/': - goto BlockCommentBegin; - } + context.Position++; + return true; } + return false; + } - goto Finish; - - BlockCommentEnd: - while (currCharOfs < end) + private static void HandleBlockComment(string sql, ParseContext context, char ch) + { + switch (ch) { - ch = sql[currCharOfs++]; - if (ch == '/') + case '/' when context.Position < sql.Length && sql[context.Position] == '*': + context.Position++; + context.BlockCommentLevel++; + break; + case '*' when context.Position < sql.Length && sql[context.Position] == '/': { - if (--blockCommentLevel > 0) - goto BlockComment; - goto None; + context.Position++; + context.BlockCommentLevel--; + if (context.BlockCommentLevel == 0) + context.State = ParseState.Normal; + break; } - - if (ch != '*') - goto BlockComment; } + } - goto Finish; + private static string ExtractStatement(string sql, ParseContext context) + { + var statementLength = context.Position - context.StatementStart - 1; + return statementLength <= 0 ? string.Empty : sql.Substring(context.StatementStart, statementLength); + } - SemiColon: - currentStatementBuilder.Append(sql, currTokenBeg, currCharOfs - currTokenBeg - 1); - result.Add(currentStatementBuilder.ToString()); - while (currCharOfs < end) + private static void SkipWhitespaceAndStartNext(string sql, ParseContext context) + { + while (context.Position < sql.Length && char.IsWhiteSpace(sql[context.Position])) { - ch = sql[currCharOfs]; - if (char.IsWhiteSpace(ch)) - { - currCharOfs++; - continue; - } - - currentStatementBuilder.Clear(); - - currTokenBeg = currCharOfs; - goto None; + context.Position++; } - - return result; - - Finish: - currentStatementBuilder.Append(sql, currTokenBeg, end - currTokenBeg); - result.Add(currentStatementBuilder.ToString()); - return result; + context.StatementStart = context.Position; } } diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index 10c98f3..c35726a 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -6,12 +6,10 @@ DbUp Contributors DbUp Copyright © DbUp Contributors 2015 - netstandard2.0 + netstandard2.1;net462;net48;net8.0;net9.0 dbup-clickhouse DbUp.ClickHouse dbup-clickhouse - ../dbup.snk - true https://github.com/DbUp/dbup-clickhouse.git dbup-icon.png From e071b86a0631b8dcf4c77752a8c8d44f8a41275d Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Tue, 9 Sep 2025 17:40:45 +0500 Subject: [PATCH 03/17] Refactor `ClickHouseQueryParser` for improved readability and formatting adjustments --- src/dbup-clickhouse/ClickHouseQueryParser.cs | 39 ++++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/dbup-clickhouse/ClickHouseQueryParser.cs b/src/dbup-clickhouse/ClickHouseQueryParser.cs index 7212b47..3130e4d 100644 --- a/src/dbup-clickhouse/ClickHouseQueryParser.cs +++ b/src/dbup-clickhouse/ClickHouseQueryParser.cs @@ -4,7 +4,7 @@ namespace DbUp.ClickHouse; /// -/// Provides SQL query parsing functionality for ClickHouse scripts, capable of splitting multi-statement SQL +/// Provides SQL query parsing functionality for ClickHouse scripts, capable of splitting multi-statement SQL /// into individual statements while properly handling comments, string literals, and quoted identifiers. /// internal static class ClickHouseQueryParser @@ -16,29 +16,37 @@ private enum ParseState { /// Normal SQL parsing state. Normal, + /// Inside a single-quoted string literal. SingleQuote, + /// Inside a back-tick quoted identifier. BackTickQuote, + /// Inside a line comment (-- style). LineComment, + /// Inside a block comment (/* */ style). - BlockComment + BlockComment, } /// /// Maintains parsing context and position information during SQL processing. /// - private struct ParseContext() + private record ParseContext { /// Current character position in the SQL string. - public int Position { get; set; } = 0; - /// Starting position of the current SQL statement being parsed. - public int StatementStart { get; set; } = 0; + public int Position { get; set; } + + /// The starting position of the current SQL statement being parsed. + public int StatementStart { get; set; } + /// Current nesting level of parentheses to avoid splitting on semicolons within function calls or subqueries. - public int ParenthesisLevel { get; set; } = 0; + public int ParenthesisLevel { get; set; } + /// Current nesting level of block comments to handle nested /* */ comments correctly. - public int BlockCommentLevel { get; set; } = 0; + public int BlockCommentLevel { get; set; } + /// Current parsing state indicating the type of content being processed. public ParseState State { get; set; } = ParseState.Normal; } @@ -67,7 +75,7 @@ public static IReadOnlyCollection ParseRawQuery(string sql) var statements = new List(); var context = new ParseContext(); - + while (context.Position < sql.Length) { if (TryParseStatement(sql, context, out var statement)) @@ -89,7 +97,7 @@ public static IReadOnlyCollection ParseRawQuery(string sql) private static bool TryParseStatement(string sql, ParseContext context, out string statement) { statement = string.Empty; - + while (context.Position < sql.Length) { var ch = sql[context.Position]; @@ -109,7 +117,12 @@ private static bool TryParseStatement(string sql, ParseContext context, out stri return false; } - private static bool HandleNormalState(string sql, ParseContext context, char ch, out string statement) + private static bool HandleNormalState( + string sql, + ParseContext context, + char ch, + out string statement + ) { statement = string.Empty; @@ -213,7 +226,9 @@ private static void HandleBlockComment(string sql, ParseContext context, char ch private static string ExtractStatement(string sql, ParseContext context) { var statementLength = context.Position - context.StatementStart - 1; - return statementLength <= 0 ? string.Empty : sql.Substring(context.StatementStart, statementLength); + return statementLength <= 0 + ? string.Empty + : sql.Substring(context.StatementStart, statementLength); } private static void SkipWhitespaceAndStartNext(string sql, ParseContext context) From a71230bbea268b801071bfb4461fd7d93482a262 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Fri, 12 Sep 2025 12:35:48 +0500 Subject: [PATCH 04/17] ci: Added gitlab CI pipeline --- .gitlab-ci.yml | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..2272e7e --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,85 @@ +variables: + # Opt out of telemetry (turbo and others), see https://consoledonottrack.com/ + DO_NOT_TRACK: 1 + # Enable debug services + CI_DEBUG_SERVICES: "true" + # Enable service network + FF_NETWORK_PER_BUILD: "true" + # Nupkg folder + NUPKG_FOLDER: ".packages" + NUGET_SOURCE: https://nuget.rapidsoft.ru/nuget/Rapidsoft-Nuget/v3/index.json + NUGETORG_SOURCE: https://nuget.rapidsoft.ru/nuget/RapidSoft-NuGetORG/v3/index.json + GITHUB_SOURCE: https://nuget.pkg.github.com/DbUp/index.json + # All console output of dotnet should be in English + LANG: C + LCID: 1033 + # Disable dotnet first time experience + DOTNET_NOLOGO: true + # Disable Husky in dotnet tools restore + HUSKY: 0 + +stages: + - build + - test + - publish + - release + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG' + when: never + - if: $CI_COMMIT_BRANCH + when: always + +include: + - component: $CI_SERVER_FQDN/open-source/ci-components/ultimate-auto-semversioning@0.1.3 + +build: + image: mcr.microsoft.com/dotnet/sdk:8.0 + stage: build + variables: + PACKAGE_VERSION: ${GitVersion_LegacySemVer} + script: + - | + echo "Building version ${PACKAGE_VERSION}" + if [[ "${CI_SCRIPT_TRACE}" == "true" ]] || [[ -n "${CI_DEBUG_TRACE}" ]]; then + echo "Debugging enabled" + set -xv + fi + dotnet nuget update source nuget.org -s "$NUGETORG_SOURCE" + dotnet nuget add source "$NUGET_SOURCE" --name nuget.rapidsoft.ru + dotnet nuget add source "$GITHUB_SOURCE" --name github --username shokurov --password $GITHUB_REPO_KEY --store-password-in-clear-text + + dotnet restore --packages .nuget/packages/ ./src/dbup-clickhouse.sln + dotnet build --no-restore --packages .nuget/packages/ -c Release -p:Version=$PACKAGE_VERSION ./src/dbup-clickhouse.sln + dotnet pack --no-build --no-restore -c Release -p:Version=$PACKAGE_VERSION -o $NUPKG_FOLDER --include-symbols --include-source ./src/dbup-clickhouse.sln + artifacts: + expire_in: 1 week # to save gitlab server space, we copy the files we need to deploy folder later on + paths: + - "$NUPKG_FOLDER/" + +publish:internal: + image: mcr.microsoft.com/dotnet/sdk:8.0 + stage: publish + variables: + PACKAGE_VERSION: ${GitVersion_LegacySemVer} + script: + - echo "Publishing version $PACKAGE_VERSION internally to $NUGET_SOURCE" + - dotnet nuget push "$NUPKG_FOLDER/*.nupkg" --source $NUGET_SOURCE --api-key $NUGET_API_KEY + needs: + - build + when: manual + +# Please see [Release CI/CD examples](https://docs.gitlab.com/ee/user/project/releases/release_cicd_examples.html) for many rich examples for triggering a release only on specific conditions. +release_job: + stage: release + when: manual + image: registry.gitlab.com/gitlab-org/release-cli + rules: + - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED != "false"' + #This last rule will do a release for any protected branch. Literal branch names can be used if desired. + script: + - echo "running release_job" + release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties + tag_name: '$CI_COMMIT_TAG' + description: '$CI_COMMIT_TAG' From 0640ddcd0435d39078f3ee92a10e77db6bf9bae8 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Fri, 12 Sep 2025 12:41:44 +0500 Subject: [PATCH 05/17] ci: Update .NET image to 9.0 and adjust solution file entries --- .gitlab-ci.yml | 2 +- src/dbup-clickhouse.sln | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2272e7e..8b4e2b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,7 +59,7 @@ build: - "$NUPKG_FOLDER/" publish:internal: - image: mcr.microsoft.com/dotnet/sdk:8.0 + image: mcr.microsoft.com/dotnet/sdk:9.0 stage: publish variables: PACKAGE_VERSION: ${GitVersion_LegacySemVer} diff --git a/src/dbup-clickhouse.sln b/src/dbup-clickhouse.sln index 49a274f..4ed19cc 100644 --- a/src/dbup-clickhouse.sln +++ b/src/dbup-clickhouse.sln @@ -10,6 +10,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_", "_", "{2BB18839-D96E-46 ..\.gitignore = ..\.gitignore ..\license.txt = ..\license.txt ..\README.md = ..\README.md + .editorconfig = .editorconfig + ..\.gitlab-ci.yml = ..\.gitlab-ci.yml EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{62E5FE92-E288-4E09-964D-F92AF0E49131}" @@ -24,12 +26,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{77157734-01D .editorconfig = .editorconfig EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".Solution Items", ".Solution Items", "{CC2BF5D2-DA01-4524-BA87-488F9C164356}" - ProjectSection(SolutionItems) = preProject - .editorconfig = .editorconfig - Directory.Build.props = Directory.Build.props - EndProjectSection -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU From 144defaf17091efc19847699eace8582bb41a7f9 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Fri, 12 Sep 2025 13:30:28 +0500 Subject: [PATCH 06/17] ci: Update .NET build image to 9.0 in GitLab CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b4e2b1..9cbe965 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ include: - component: $CI_SERVER_FQDN/open-source/ci-components/ultimate-auto-semversioning@0.1.3 build: - image: mcr.microsoft.com/dotnet/sdk:8.0 + image: mcr.microsoft.com/dotnet/sdk:9.0 stage: build variables: PACKAGE_VERSION: ${GitVersion_LegacySemVer} From 405aa1abddb0d17f99e87d5d8072ec8134415e37 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Fri, 12 Sep 2025 13:54:38 +0500 Subject: [PATCH 07/17] ci: Update release job image and use PACKAGE_VERSION for tags --- .gitlab-ci.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9cbe965..e860a65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -74,12 +74,11 @@ publish:internal: release_job: stage: release when: manual - image: registry.gitlab.com/gitlab-org/release-cli - rules: - - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED != "false"' - #This last rule will do a release for any protected branch. Literal branch names can be used if desired. + image: registry.gitlab.com/gitlab-org/cli:latest + variables: + PACKAGE_VERSION: ${GitVersion_LegacySemVer} script: - echo "running release_job" release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties - tag_name: '$CI_COMMIT_TAG' - description: '$CI_COMMIT_TAG' + tag_name: '$PACKAGE_VERSION' + description: '$PACKAGE_VERSION' From dabf702ef373ebb70f177f12faaba12d9a73cd55 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Thu, 25 Sep 2025 14:28:45 +0500 Subject: [PATCH 08/17] ci: Downgrade dbup-core package to version 6.0.4 --- src/dbup-clickhouse/dbup-clickhouse.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index c35726a..46fa69e 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -16,7 +16,7 @@ - + From acf89da3267a4bb6173968ffd2e8113d010e5760 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Tue, 30 Sep 2025 18:23:44 +0300 Subject: [PATCH 09/17] fix: add .NET 6.0 support and resolve package version conflicts --- src/dbup-clickhouse/dbup-clickhouse.csproj | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index 46fa69e..9e5d771 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -6,7 +6,7 @@ DbUp Contributors DbUp Copyright © DbUp Contributors 2015 - netstandard2.1;net462;net48;net8.0;net9.0 + netstandard2.1;net462;net48;net6.0;net8.0;net9.0 dbup-clickhouse DbUp.ClickHouse dbup-clickhouse @@ -17,6 +17,21 @@ + + + + + + + + + + + + + + + From f1bbb28118377f3c73fca234307d914231bbb11b Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Tue, 30 Sep 2025 19:13:30 +0300 Subject: [PATCH 10/17] Revert "fix: add .NET 6.0 support and resolve package version conflicts" This reverts commit acf89da3267a4bb6173968ffd2e8113d010e5760. --- src/dbup-clickhouse/dbup-clickhouse.csproj | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index 9e5d771..46fa69e 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -6,7 +6,7 @@ DbUp Contributors DbUp Copyright © DbUp Contributors 2015 - netstandard2.1;net462;net48;net6.0;net8.0;net9.0 + netstandard2.1;net462;net48;net8.0;net9.0 dbup-clickhouse DbUp.ClickHouse dbup-clickhouse @@ -17,21 +17,6 @@ - - - - - - - - - - - - - - - From 7d39af87e2e4a5b2499242a1144ac787d54b2474 Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Wed, 5 Nov 2025 10:38:30 +0500 Subject: [PATCH 11/17] fix(PR): Restored package signing See @droyad comment on PR https://github.com/DbUp/dbup-clickhouse/pull/2#discussion_r2492791995 --- src/dbup-clickhouse/dbup-clickhouse.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index 46fa69e..dadb83f 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -10,6 +10,8 @@ dbup-clickhouse DbUp.ClickHouse dbup-clickhouse + ../dbup.snk + true https://github.com/DbUp/dbup-clickhouse.git dbup-icon.png From 10b25fa94f3f0c84b1cea0472ceb267dfac43b9a Mon Sep 17 00:00:00 2001 From: Egor Shokurov Date: Wed, 5 Nov 2025 10:48:08 +0500 Subject: [PATCH 12/17] fix(PR): Removed excess target frameworks and reverted Assembly signing Assembly signing failed because of error CS8002: Referenced assembly 'ClickHouse.Driver, Version=0.7.20.0, Culture=neutral, PublicKeyToken=null' does not have a strong name --- src/dbup-clickhouse/dbup-clickhouse.csproj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index dadb83f..264afc8 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -6,12 +6,10 @@ DbUp Contributors DbUp Copyright © DbUp Contributors 2015 - netstandard2.1;net462;net48;net8.0;net9.0 + netstandard2.1;net462 dbup-clickhouse DbUp.ClickHouse dbup-clickhouse - ../dbup.snk - true https://github.com/DbUp/dbup-clickhouse.git dbup-icon.png From 3f2c8869c968a23d894cfd9a4b8133694e425cea Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Thu, 6 Nov 2025 11:12:35 +1000 Subject: [PATCH 13/17] Used latest workflow --- .github/workflows/main.yml | 73 +++----------------------------------- 1 file changed, 4 insertions(+), 69 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 18198a1..4deded3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,77 +2,12 @@ name: CI on: push: + branches: + - '**' # Ignores pushes of tags pull_request: workflow_dispatch: jobs: build: - runs-on: windows-latest # Use Ubuntu in v5.0 - - env: - DOTNET_NOLOGO: true - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true # Avoid pre-populating the NuGet package cache - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # all - - - name: Setup .NET 2.0 # Remove in v5.0 - uses: actions/setup-dotnet@v1 - with: - dotnet-version: 2.0.x - - - name: Setup .NET 8.0 - uses: actions/setup-dotnet@v1 - with: - dotnet-version: 8.0.x - - - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v0 - with: - versionSpec: '5.x' - - - name: Run GitVersion - id: gitversion - uses: gittools/actions/gitversion/execute@v0 - - - name: Display SemVer - run: | - echo "SemVer: $env:GitVersion_SemVer" - - - name: Add DbUp NuGet Source - run: dotnet nuget add source --name DbUp --username DbUp --password ${{ secrets.GITHUB_TOKEN }} --store-password-in-clear-text https://nuget.pkg.github.com/DbUp/index.json - - - name: Restore - run: dotnet restore - working-directory: src - - - name: Build - run: dotnet build -c Release --no-restore /p:Version=$env:GitVersion_SemVer - working-directory: src - - - name: Test - run: dotnet test --no-build -c Release --logger trx --logger "console;verbosity=detailed" --results-directory ../artifacts - working-directory: src - - - name: Pack - run: dotnet pack --no-build -c Release -o ../artifacts /p:Version=$env:GitVersion_SemVer - working-directory: src - - - name: Push NuGet packages to GitHub Packages ⬆️ - working-directory: artifacts - run: dotnet nuget push *.nupkg --api-key ${{ secrets.GITHUB_TOKEN }} --source "https://nuget.pkg.github.com/DbUp/index.json" - - - name: Push NuGet packages to NuGet ⬆️ - if: ${{ steps.gitversion.outputs.preReleaseLabel == '' }} - working-directory: artifacts - run: dotnet nuget push *.nupkg --api-key ${{ secrets.NUGET_APIKEY }} --source https://api.nuget.org/v3/index.json - - - name: Test Report 🧪 - uses: dorny/test-reporter@v1 - if: ${{ always() }} - with: - name: Tests - path: artifacts/*.trx - reporter: dotnet-trx + name: Build + uses: DbUp/Universe/.github/workflows/build.yml@main \ No newline at end of file From 8aa0691d0afa0efc49a24a917261728fb4c6a8e5 Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Thu, 6 Nov 2025 11:16:53 +1000 Subject: [PATCH 14/17] Include txt Approvals in the output --- src/Tests/Tests.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tests/Tests.csproj b/src/Tests/Tests.csproj index 13c29eb..2937325 100644 --- a/src/Tests/Tests.csproj +++ b/src/Tests/Tests.csproj @@ -25,6 +25,8 @@ + + From 10ee7de1e48d76cc30536a7f8ab316e4806fe41f Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Thu, 6 Nov 2025 11:32:25 +1000 Subject: [PATCH 15/17] Updated to latest libraries --- README.md | 4 +++- .../NoPublicApiChanges.Run.approved.cs | 15 ++++++++------- src/Tests/Tests.csproj | 10 ++++------ src/dbup-clickhouse/dbup-clickhouse.csproj | 2 +- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 59b2abf..65d6df6 100644 --- a/README.md +++ b/README.md @@ -13,4 +13,6 @@ Please only log issue related to ClickHouse support in this repo. For cross cutt # Contributing -See the [readme in our main repo](https://github.com/DbUp/DbUp/blob/master/README.md) for how to get started and contribute. \ No newline at end of file +See the [readme in our main repo](https://github.com/DbUp/DbUp/blob/master/README.md) for how to get started and contribute. + +To run the tests, start the clickhouse container by running `./start-clickhouse.ps1` \ No newline at end of file diff --git a/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs b/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs index c7d2b61..8ae506b 100644 --- a/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs +++ b/src/Tests/ApprovalFiles/NoPublicApiChanges.Run.approved.cs @@ -1,7 +1,8 @@ - +[assembly: System.Reflection.AssemblyMetadata("RepositoryUrl", "https://github.com/DbUp/dbup-clickhouse.git")] +[assembly: System.Runtime.Versioning.TargetFramework(".NETStandard,Version=v2.1", FrameworkDisplayName=".NET Standard 2.1")] namespace DbUp.ClickHouse { - public class ClickHouseConnectionManager : DbUp.Engine.Transactions.DatabaseConnectionManager, DbUp.Engine.Transactions.IConnectionManager + public class ClickHouseConnectionManager : DbUp.Engine.Transactions.DatabaseConnectionManager { public ClickHouseConnectionManager(string connectionString) { } public override System.Collections.Generic.IEnumerable SplitScriptIntoCommands(string scriptContents) { } @@ -9,13 +10,13 @@ public override System.Collections.Generic.IEnumerable SplitScriptIntoCo public static class ClickHouseExtensions { public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(DbUp.Engine.Transactions.IConnectionManager connectionManager) { } - public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supported, string connectionString) { } public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supported, DbUp.Engine.Transactions.IConnectionManager connectionManager) { } + public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supported, string connectionString) { } public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(DbUp.Engine.Transactions.IConnectionManager connectionManager, string schema) { } public static DbUp.Builder.UpgradeEngineBuilder ClickHouseDatabase(this DbUp.Builder.SupportedDatabases supported, string connectionString, string database) { } public static DbUp.Builder.UpgradeEngineBuilder JournalToClickHouseTable(this DbUp.Builder.UpgradeEngineBuilder builder, string schema, string table) { } } - public class ClickHouseJournal : DbUp.Support.TableJournal, DbUp.Engine.IJournal + public class ClickHouseJournal : DbUp.Support.TableJournal { public ClickHouseJournal(System.Func connectionManager, System.Func logger, string schema, string tableName) { } protected override string CreateSchemaTableSql(string quotedPrimaryKeyName) { } @@ -23,7 +24,7 @@ protected override string DoesTableExistSql() { } protected override string GetInsertJournalEntrySql(string scriptName, string applied) { } protected override string GetJournalEntriesSql() { } } - public class ClickHouseObjectParser : DbUp.Support.SqlObjectParser, DbUp.Engine.ISqlObjectParser + public class ClickHouseObjectParser : DbUp.Support.SqlObjectParser { public ClickHouseObjectParser() { } } @@ -32,10 +33,10 @@ public class ClickHousePreprocessor : DbUp.Engine.IScriptPreprocessor public ClickHousePreprocessor() { } public string Process(string contents) { } } - public class ClickHouseScriptExecutor : DbUp.Support.ScriptExecutor, DbUp.Engine.IScriptExecutor + public class ClickHouseScriptExecutor : DbUp.Support.ScriptExecutor { public ClickHouseScriptExecutor(System.Func connectionManagerFactory, System.Func log, string schema, System.Func variablesEnabled, System.Collections.Generic.IEnumerable scriptPreprocessors, System.Func journalFactory) { } protected override void ExecuteCommandsWithinExceptionHandler(int index, DbUp.Engine.SqlScript script, System.Action executeCommand) { } protected override string GetVerifySchemaSql(string schema) { } } -} +} \ No newline at end of file diff --git a/src/Tests/Tests.csproj b/src/Tests/Tests.csproj index 2937325..cd735f2 100644 --- a/src/Tests/Tests.csproj +++ b/src/Tests/Tests.csproj @@ -11,11 +11,11 @@ - - - + + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -25,8 +25,6 @@ - - diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index 264afc8..6c70211 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -16,7 +16,7 @@ - + From 8272c9a804f8599080082e81da42313daa72904c Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Thu, 6 Nov 2025 11:39:29 +1000 Subject: [PATCH 16/17] Get approvals to work --- src/Directory.Build.props | 13 ------------- src/dbup-clickhouse/dbup-clickhouse.csproj | 7 +++++++ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 011b551..4527be4 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -11,17 +11,4 @@ true - - - - true - - - true - - - embedded - - - diff --git a/src/dbup-clickhouse/dbup-clickhouse.csproj b/src/dbup-clickhouse/dbup-clickhouse.csproj index 6c70211..bb46a8e 100644 --- a/src/dbup-clickhouse/dbup-clickhouse.csproj +++ b/src/dbup-clickhouse/dbup-clickhouse.csproj @@ -14,6 +14,13 @@ dbup-icon.png + + true + true + embedded + + + From eccd68d46182db40244ef73a2c42371db85dc9b7 Mon Sep 17 00:00:00 2001 From: Robert Wagner Date: Thu, 6 Nov 2025 11:42:13 +1000 Subject: [PATCH 17/17] Added missing templates --- .github/workflows/publish-release.yml | 12 ++++++++++++ .github/workflows/test-report.yml | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 .github/workflows/publish-release.yml create mode 100644 .github/workflows/test-report.yml diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 0000000..eb5e03f --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,12 @@ +name: Publish DbUp Packages to NuGet + +on: + release: + types: [ published ] + workflow_dispatch: + +jobs: + publish: + name: Publish Package + uses: DbUp/Universe/.github/workflows/publish-release.yml@main + secrets: inherit diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml new file mode 100644 index 0000000..e8c9f8f --- /dev/null +++ b/.github/workflows/test-report.yml @@ -0,0 +1,12 @@ +name: Test Report +run-name: Generate Test Report for run `${{ github.event.workflow_run.run_number }}` branch `${{ github.event.workflow_run.head_branch }}` + +on: + workflow_run: + workflows: ["CI", "build"] + types: [completed] + +jobs: + report: + name: Test Report 🧪 + uses: DbUp/Universe/.github/workflows/test-report.yml@main \ No newline at end of file