Merge pull request brooklyn-data#444 from KijkEr/add_sql_server_support

Add sql server support
mobsuccess-devops · Sep 25, 2024 · d06f20f · d06f20f
2 parents 5e824d2 + 77872fd
commit d06f20f
Show file tree

Hide file tree

Showing 45 changed files with 1,269 additions and 737 deletions.
diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml
@@ -113,7 +113,7 @@ jobs:
     strategy:
       fail-fast: false # Don't fail one DWH if the others fail
       matrix:
-        warehouse: ["snowflake", "bigquery", "postgres"]
+        warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"]
         # When supporting a new version, update the list here
         version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"]
     runs-on: ubuntu-latest
@@ -145,6 +145,12 @@ jobs:
       - name: Install tox
         run: python3 -m pip install tox
 
+      - name: Install SQL Server
+        run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=123" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2022-latest
+
+      - name: Install Microsoft ODBC
+        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+
       - name: Checkout
         uses: actions/checkout@v3
         with:

diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml
@@ -34,7 +34,7 @@ jobs:
   integration:
     strategy:
       matrix:
-        warehouse: ["snowflake", "bigquery", "postgres"]
+        warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"]
         version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"]
     runs-on: ubuntu-latest
     permissions:
@@ -66,6 +66,12 @@ jobs:
       - name: Install tox
         run: python3 -m pip install tox
 
+      - name: Install SQL Server
+        run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=123" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2022-latest
+
+      - name: Install Microsoft ODBC
+        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+
       - id: auth
         if: ${{ matrix.warehouse == 'bigquery' }}
         uses: google-github-actions/auth@v1

diff --git a/README.md b/README.md
@@ -19,6 +19,7 @@ The package currently supports
 - Snowflake :white_check_mark:
 - Google BigQuery :white_check_mark:
 - Postgres :white_check_mark:
+- SQL Server :white_check_mark:
 
 Models included:
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -15,8 +15,9 @@ models:
     +file_format: delta
     sources:
       +materialized: incremental
-      +on_schema_change: append_new_columns
       +full_refresh: false
       +persist_docs:
-        # Databricks doesn't offer column-level support for persisting docs
-        columns: '{{ target.name != "databricks" }}'
+        # Databricks and SQL Server don't offer column-level support for persisting docs
+        columns: '{{ target.name != "databricks" and target.type != "sqlserver" }}'
+        relation: '{{ target.type != "sqlserver" }}'
+      +as_columnstore: False
diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml
@@ -25,8 +25,8 @@ vars:
 
 models:
   +persist_docs:
-    relation: true
-    columns: true
+    relation: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this
+    columns: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this
 seeds:
   +quote_columns: false
 

diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml
@@ -52,3 +52,14 @@ dbt_artifacts:
       dbname: postgres
       schema: public
       threads: 8
+    sqlserver:
+      type: sqlserver
+      driver: 'ODBC Driver 18 for SQL Server'
+      server: localhost
+      port: 1433
+      database: dbt_artifact_integrationtests
+      schema: dbo
+      windows_login: False
+      trust_cert: True
+      user: sa
+      password: "123"
diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql
@@ -1 +1,2 @@
-select 1 as failures from (select 2) as foo where 1 = 2
+select 1 as failures from (select 2 as two) as foo where 1 = 2
+
diff --git a/macros/migration/migrate_from_v0_to_v1.sql b/macros/migration/migrate_from_v0_to_v1.sql
@@ -9,7 +9,7 @@
             node_id,
             query_completed_at,
             rows_affected,
-            schema,
+            "schema",
             status,
             thread_id,
             total_node_runtime,
@@ -34,9 +34,7 @@
     {% endset %}
 
     {{ log("Migrating model_executions", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_model_executions }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_model_executions }} {%- endcall -%}
 
     {% set migrate_tests %}
         insert into {{new_database}}.{{new_schema}}.tests (
@@ -62,9 +60,7 @@
     {% endset %}
 
     {{ log("Migrating tests", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_tests }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_tests }} {%- endcall -%}
 
     {% set migrate_test_executions %}
         insert into {{new_database}}.{{new_schema}}.test_executions (
@@ -96,22 +92,20 @@
     {% endset %}
 
     {{ log("Migrating test_executions", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_test_executions }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_test_executions }} {%- endcall -%}
 
     {% set migrate_models %}
         insert into {{new_database}}.{{new_schema}}.models (
             checksum,
             command_invocation_id,
-            database,
+            "database",
             depends_on_nodes,
             materialization,
             name,
             node_id,
             package_name,
             path,
-            schema,
+            "schema",
             run_started_at
         )
         select
@@ -130,20 +124,18 @@
     {% endset %}
 
     {{ log("Migrating models", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_models }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_models }} {%- endcall -%}
 
     {% set migrate_seeds %}
         insert into {{new_database}}.{{new_schema}}.seeds (
             checksum,
             command_invocation_id,
-            database,
+            "database",
             name,
             node_id,
             package_name,
             path,
-            schema,
+            "schema",
             run_started_at
         )
         select
@@ -160,9 +152,7 @@
     {% endset %}
 
     {{ log("Migrating seeds", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_seeds }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_seeds }} {%- endcall -%}
 
     {% set migrate_seed_executions %}
         insert into {{new_database}}.{{new_schema}}.seed_executions (
@@ -173,7 +163,7 @@
             node_id,
             query_completed_at,
             rows_affected,
-            schema,
+            "schema",
             status,
             thread_id,
             total_node_runtime,
@@ -198,9 +188,7 @@
     {% endset %}
 
     {{ log("Migrating seed_executions", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_seed_executions }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_seed_executions }} {%- endcall -%}
 
     {% set migrate_exposures %}
         insert into {{new_database}}.{{new_schema}}.exposures (
@@ -235,21 +223,19 @@
     {% endset %}
 
     {{ log("Migrating exposures", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_exposures }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_exposures }} {%- endcall -%}
 
     {% set migrate_snapshots %}
         insert into {{new_database}}.{{new_schema}}.snapshots (
             checksum,
             command_invocation_id,
-            database,
+            "database",
             depends_on_nodes,
             name,
             node_id,
             package_name,
             path,
-            schema,
+            "schema",
             strategy,
             run_started_at
         )
@@ -269,9 +255,7 @@
     {% endset %}
 
     {{ log("Migrating snapshots", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_snapshots }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_snapshots }} {%- endcall -%}
 
     {% set migrate_snapshot_executions %}
         insert into {{new_database}}.{{new_schema}}.snapshot_executions (
@@ -282,7 +266,7 @@
             node_id,
             query_completed_at,
             rows_affected,
-            schema,
+            "schema",
             status,
             thread_id,
             total_node_runtime,
@@ -307,21 +291,19 @@
     {% endset %}
 
     {{ log("Migrating snapshot_executions", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_snapshot_executions }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_snapshot_executions }} {%- endcall -%}
 
     {% set migrate_sources %}
         insert into {{new_database}}.{{new_schema}}.sources (
             command_invocation_id,
-            database,
+            "database",
             freshness,
             identifier,
             loaded_at_field,
             loader,
             name,
             node_id,
-            schema,
+            "schema",
             source_name,
             run_started_at
         )
@@ -341,9 +323,8 @@
     {% endset %}
 
     {{ log("Migrating sources", info=True) }}
-    {%- call statement(auto_begin=True) -%}
-        {{ migrate_sources }}
-    {%- endcall -%}
+    {%- call statement(auto_begin=True) -%} {{ migrate_sources }} {%- endcall -%}
 
     {{ log("Migration complete. You can now safely delete any data from before 1.0.0", info=True) }}
 {%- endmacro -%}
+
diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql
@@ -1,5 +1,5 @@
 {% macro upload_exposures(exposures) -%}
-    {{ return(adapter.dispatch('get_exposures_dml_sql', 'dbt_artifacts')(exposures)) }}
+    {{ return(adapter.dispatch("get_exposures_dml_sql", "dbt_artifacts")(exposures)) }}
 {%- endmacro %}
 
 {% macro default__get_exposures_dml_sql(exposures) -%}
@@ -47,8 +47,7 @@
         {%- endfor %}
         {% endset %}
         {{ exposure_values }}
-    {% else %}
-        {{ return("") }}
+    {% else %} {{ return("") }}
     {% endif %}
 {% endmacro -%}
 
@@ -80,8 +79,7 @@
             {%- endfor %}
         {% endset %}
         {{ exposure_values }}
-    {% else %}
-        {{ return("") }}
+    {% else %} {{ return("") }}
     {% endif %}
 {%- endmacro %}
 
@@ -114,7 +112,45 @@
             {%- endfor %}
         {% endset %}
         {{ exposure_values }}
-    {% else %}
-        {{ return("") }}
+    {% else %} {{ return("") }}
     {% endif %}
 {%- endmacro %}
+
+{% macro sqlserver__get_exposures_dml_sql(exposures) -%}
+
+    {% if exposures != [] %}
+        {% set exposure_values %}
+        select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14"
+        from ( values
+        {% for exposure in exposures -%}
+            (
+                '{{ invocation_id }}', {# command_invocation_id #}
+                '{{ exposure.unique_id | replace("'","''") }}', {# node_id #}
+                '{{ run_started_at }}', {# run_started_at #}
+                '{{ exposure.name | replace("'","''") }}', {# name #}
+                '{{ exposure.type }}', {# type #}
+                '{{ tojson(exposure.owner) }}', {# owner #}
+                '{{ exposure.maturity }}', {# maturity #}
+                '{{ exposure.original_file_path }}', {# path #}
+                '{{ exposure.description | replace("'","''") }}', {# description #}
+                '{{ exposure.url }}', {# url #}
+                '{{ exposure.package_name }}', {# package_name #}
+                '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #}
+                '{{ tojson(exposure.tags) }}', {# tags #}
+                {% if var('dbt_artifacts_exclude_all_results', false) %}
+                    null
+                {% else %}
+                    '{{ tojson(exposure) | replace("'", "''") }}' {# all_results #}
+                {% endif %}
+            )
+            {%- if not loop.last %},{%- endif %}
+        {%- endfor %}
+
+        ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14")
+
+        {% endset %}
+        {{ exposure_values }}
+    {% else %} {{ return("") }}
+    {% endif %}
+{% endmacro -%}
+