Merge pull request #2 from fivetran/docs-update

Docs update

.circleci/config.yml

@@ -28,17 +28,27 @@ jobs:
             echo `pwd`
             cd integration_tests
             dbt deps
-            dbt seed --target redshift --full-refresh --exclude email_event_dropped_data_snowflake email_event_sent_data_snowflake
+            dbt seed --target redshift --full-refresh
             dbt run --target redshift --full-refresh
             dbt test --target redshift
+      - run:
+          name: "Run Tests - Postgres"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target postgres --full-refresh
+            dbt run --target postgres --full-refresh
+            dbt test --target postgres
       - run:
           name: "Run Tests - Snowflake"
           command: |
             . venv/bin/activate
             echo `pwd`
             cd integration_tests
             dbt deps
-            dbt seed --target snowflake --full-refresh --exclude email_event_dropped_data email_event_sent_data
+            dbt seed --target snowflake --full-refresh
             dbt run --target snowflake --full-refresh
             dbt test --target snowflake
       - run:
@@ -51,6 +61,6 @@ jobs:
             echo `pwd`
             cd integration_tests
             dbt deps
-            dbt seed --target bigquery --full-refresh --exclude email_event_dropped_data_snowflake email_event_sent_data_snowflake
+            dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
             dbt test --target bigquery

README.md

@@ -1,4 +1,4 @@
-# Pardot Source ([docs](https://fivetran-dbt-pardot.netlify.app/#!/overview))
+# Pardot (Source)
 
 This package models Pardot data from [Fivetran's connector](https://fivetran.com/docs/applications/pardot). It uses data in the format described by [the Pardot ERD](https://fivetran.com/docs/applications/pardot#schemainformation).
 
@@ -52,6 +52,21 @@ vars:
     prospect_passthrough_columns: ["custom_creative","custom_contact_state"]
 ```
 
+### Changing the Build Schema
+By default this package will build the Pardot staging models within a schema titled (<target_schema> + `_stg_pardot`). If this is not where you would like your Pardot staging models to be written to, add the following configuration to your `dbt_project.yml` file:
+
+```yml
+# dbt_project.yml
+
+...
+models:
+  pardot_source:
+    +schema: my_new_staging_models_schema # leave blank for just the target_schema
+```
+
+## Database Support
+This package has been tested on BigQuery, Snowflake, Redshift, and Postgres.
+
 ## Contributions
 
 Additional contributions to this package are very welcome! Please create issues
@@ -60,8 +75,11 @@ or open PRs against `master`. Check out
 on the best workflow for contributing to a package.
 
 ## Resources:
+- Provide [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next
+- Have questions, feedback, or need help? Book a time during our office hours [using Calendly](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or email us at [email protected]
 - Find all of Fivetran's pre-built dbt packages in our [dbt hub](https://hub.getdbt.com/fivetran/)
-- Learn more about Fivetran [in the Fivetran docs](https://fivetran.com/docs)
+- Learn how to orchestrate [dbt transformations with Fivetran](https://fivetran.com/docs/transformations/dbt)
+- Learn more about Fivetran overall [in our docs](https://fivetran.com/docs)
 - Check out [Fivetran's blog](https://fivetran.com/blog)
 - Learn more about dbt [in the dbt docs](https://docs.getdbt.com/docs/introduction)
 - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers

dbt_project.yml

@@ -1,16 +1,19 @@
 name: 'pardot_source'
 version: '0.2.0'
 config-version: 2
-require-dbt-version: [">=0.18.0", "<0.19.0"]
+require-dbt-version: [">=0.18.0", "<0.20.0"]
 
+models:
+  pardot_source:
+    +schema: stg_pardot
 vars:
   pardot_source:
-    list: "{{ source('pardot','list') }}"
-    list_membership: "{{ source('pardot','list_membership') }}"
-    visitor: "{{ source('pardot','visitor') }}"
-    visitor_activity: "{{ source('pardot','visitor_activity') }}"
-    prospect: "{{ source('pardot','prospect') }}"
-    campaign: "{{ source('pardot','campaign') }}"
-    opportunity: "{{ source('pardot','opportunity') }}"
+    list:                 "{{ source('pardot','list') }}"
+    list_membership:      "{{ source('pardot','list_membership') }}"
+    visitor:              "{{ source('pardot','visitor') }}"
+    visitor_activity:     "{{ source('pardot','visitor_activity') }}"
+    prospect:             "{{ source('pardot','prospect') }}"
+    campaign:             "{{ source('pardot','campaign') }}"
+    opportunity:          "{{ source('pardot','opportunity') }}"
     opportunity_prospect: "{{ source('pardot','opportunity_prospect') }}"
     prospect_passthrough_columns: []

integration_tests/ci/sample.profiles.yml

@@ -18,6 +18,15 @@ integration_tests:
       port: 5439
       schema: pardot_source_integration_tests
       threads: 8
+    postgres:
+      type: postgres
+      host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}"
+      user: "{{ env_var('CI_POSTGRES_DBT_USER') }}"
+      pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
+      dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
+      port: 5432
+      schema: pardot_source_integration_tests
+      threads: 8
     bigquery:
       type: bigquery
       method: service-account

integration_tests/dbt_project.yml

@@ -1,15 +1,18 @@
 name: 'pardot_source_integration_tests'
-version: '1.0.0'
+version: '0.2.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
   pardot_source:
-    list: "{{ ref('pardot_list_data') }}"
-    list_membership: "{{ ref('pardot_list_membership_data') }}"
-    visitor: "{{ ref('pardot_visitor_data') }}"
-    visitor_activity: "{{ ref('pardot_visitor_activity_data') }}"
-    prospect: "{{ ref('pardot_prospect_data') }}"
-    campaign: "{{ ref('pardot_campaign_data') }}"
-    opportunity: "{{ ref('pardot_opportunity_data') }}"
-    opportunity_prospect: "{{ ref('pardot_opportunity_prospect_data') }}"
+    list:                   "{{ ref('pardot_list_data') }}"
+    list_membership:        "{{ ref('pardot_list_membership_data') }}"
+    visitor:                "{{ ref('pardot_visitor_data') }}"
+    visitor_activity:       "{{ ref('pardot_visitor_activity_data') }}"
+    prospect:               "{{ ref('pardot_prospect_data') }}"
+    campaign:               "{{ ref('pardot_campaign_data') }}"
+    opportunity:            "{{ ref('pardot_opportunity_data') }}"
+    opportunity_prospect:   "{{ ref('pardot_opportunity_prospect_data') }}"
+
+seeds:
+  +quote_columns: "{{ true if target.type == 'redshift' else false }}"

integration_tests/requirements.txt

@@ -1 +1 @@
-dbt==0.18.0
+dbt==0.19.0

models/src_pardot.yml

@@ -59,6 +59,8 @@ sources:
             description: "{{ doc('_fivetran_synced') }}"
           - name: id
             description: Pardot ID for this prospect
+          - name: user_id
+            description: Pardot ID of the user associated with this prospect
           - name: campaign_id
             description: Pardot ID of the campaign associated with this prospect
           - name: salutation
@@ -157,6 +159,8 @@ sources:
             description: "{{ doc('_fivetran_synced') }}"
           - name: id
             description: Pardot ID for this visitor
+          - name: prospect_id
+            description: Pardot ID for the associated prospect
           - name: page_view_count
             description: Number of page views by this visitor
           - name: ip_address
@@ -189,6 +193,8 @@ sources:
             description: Pardot ID for the associated prospect
           - name: visitor_id
             description: Pardot ID for the associated visitor
+          - name: opportunity_id
+            description: ID of the related opportunity
           - name: type
             description: Visitor activity's type number; See listing below
           - name: type_name
@@ -214,7 +220,7 @@ sources:
           - name: multivariate_test_variation_id
             description: Pardot ID of the multivariate test variation associated with this visitor activity
           - name: visitor_page_view_id
-            description: Pardot ID of the visitor page view associated with this visitoractivity
+            description: Pardot ID of the visitor page view associated with this visitor activity
           - name: file_id
             description: Pardot ID of the file associated with this visitor activity
           - name: custom_redirect_id

models/stg_pardot.yml

@@ -60,6 +60,8 @@ models:
         tests:
           - unique
           - not_null
+      - name: prospect_id
+        description: Pardot ID for the associated prospect
       - name: page_view_count
         description: Number of page views by this visitor
       - name: ip_address
@@ -95,6 +97,8 @@ models:
         description: Pardot ID for the associated prospect
       - name: visitor_id
         description: Pardot ID for the associated visitor
+      - name: opportunity_id
+        description: ID of the related opportunity
       - name: event_type_name
         description: Visitor activity's type name; See listing below
       - name: email_id
@@ -118,6 +122,8 @@ models:
           - not_null
       - name: campaign_id
         description: Pardot ID of the campaign associated with this prospect
+      - name: user_id
+        description: Pardot ID of the user associated with this prospect
       - name: salutation
         description: Prospect's formal prefix
       - name: first_name

packages.yml

@@ -3,5 +3,4 @@ packages:
     version: [">=0.6.0", "<0.7.0"]
 
   - git: "https://github.com/fivetran/dbt_fivetran_utils.git"
-    revision: master
     warn-unpinned: false