Add unload_table macro (#7)

* add UNLOAD macro * modifications to unload_table * remove unload from utilities * add docs; include schema arg * typo
dbt-labs · Nov 7, 2017 · bbfcd8c · bbfcd8c
1 parent 5ce4f7e
commit bbfcd8c
Show file tree

Hide file tree

Showing 2 changed files with 124 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -45,7 +45,7 @@ These views are designed to make debugging your Redshift cluster more straightfo
 
 __Introspection Models__
 
-These models (default ephemeral) make it possible to inspect tables, columns, constraints, and sort/dist keys of the Redshift cluster. These models are used to build column compression queries, but may also be generally useful. 
+These models (default ephemeral) make it possible to inspect tables, columns, constraints, and sort/dist keys of the Redshift cluster. These models are used to build column compression queries, but may also be generally useful.
 
 - [redshift_tables](models/introspection/redshift_tables.sql)
 - [redshift_columns](models/introspection/redshift_columns.sql)
@@ -81,5 +81,48 @@ Example usage:
       "{{ redshift.compress_table(this.schema, this.table, drop_backup=False) }}"
     ]
   })
-}}    
+}}
+```
+
+#### unload_table ([source](macros/unload.sql))
+
+This macro returns the SQL required to unload a Redshift table to one or more files on S3. The macro replicates all functionality provided by Redshift's [UNLOAD](http://docs.aws.amazon.com/redshift/latest/dg/r_UNLOAD.html) command.
+
+Macro signature:
+```
+{{ unload_table(schema,
+                table,
+                s3_path,
+                iam_role=None|String,
+                aws_key=None|String,
+                aws_secret=None|String,
+                manifest=Boolean,
+                delimiter=String,
+                null_as=String,
+                max_file_size=String,
+                escape=Boolean,
+                compression=None|GZIP|BZIP2,
+                add_quotes=Boolean,
+                encrypted=Boolean,
+                overwrite=Boolean,
+                parallel=Boolean) }}
+```
+
+Example usage:
+```
+{{
+  config({
+    "materialized":"table",
+    "sort": "id",
+    "dist": "id",
+    "post-hook": [
+      "{{ redshift.unload_table(this.schema,
+                                this.table,
+                                s3_path='s3://bucket/folder',
+                                aws_key='abcdef',
+                                aws_secret='ghijklm',
+                                delimiter='|') }}"
+    ]
+  })
+}}
 ```
diff --git a/macros/unload.sql b/macros/unload.sql
@@ -0,0 +1,79 @@
+-- Redshift UNLOAD grammar (see: http://docs.aws.amazon.com/redshift/latest/dg/r_UNLOAD.html)
+{#
+UNLOAD ('select-statement')
+TO 's3://object-path/name-prefix'
+authorization
+[ option [ ... ] ]
+
+where option is
+
+{ MANIFEST
+| DELIMITER [ AS ] 'delimiter-char'
+| FIXEDWIDTH [ AS ] 'fixedwidth-spec' }
+| ENCRYPTED
+| BZIP2
+| GZIP
+| ADDQUOTES
+| NULL [ AS ] 'null-string'
+| ESCAPE
+| ALLOWOVERWRITE
+| PARALLEL [ { ON | TRUE } | { OFF | FALSE } ]
+[ MAXFILESIZE [AS] max-size [ MB | GB ] ]
+
+#}
+-- Unloads a Redshift table to S3
+{% macro unload_table(schema,
+                table,
+                s3_path,
+                iam_role=None,
+                aws_key=None,
+                aws_secret=None,
+                manifest=False,
+                delimiter=",",
+                null_as="",
+                max_file_size='6 GB',
+                escape=True,
+                compression=None,
+                add_quotes=False,
+                encrypted=False,
+                overwrite=False,
+                parallel=False) %}
+
+  -- compile UNLOAD statement
+  UNLOAD ('SELECT * FROM "{{ schema }}"."{{ table }}"')
+  TO '{{ s3_path }}'
+  {% if iam_role %}
+  IAM_ROLE '{{ iam_role }}'
+  {% elif aws_key and aws_secret %}
+  ACCESS_KEY_ID '{{ aws_key }}'
+  SECRET_ACCESS_KEY '{{ aws_secret }}'
+  {% else %}
+  -- Raise an error if authorization args are not present
+  {{ exceptions.raise_compiler_error("You must provide AWS authorization parameters via 'iam_role' or 'aws_key' and 'aws_secret'.") }}
+  {% endif %}
+  {% if manifest %}
+  MANIFEST
+  {% endif %}
+  DELIMITER AS '{{ delimiter }}'
+  NULL AS '{{ null_as }}'
+  MAXFILESIZE AS {{ max_file_size }}
+  {% if escape %}
+  ESCAPE
+  {% endif %}
+  {% if compression %}
+  {{ compression|upper }}
+  {% endif %}
+  {% if add_quotes %}
+  ADDQUOTES
+  {% endif %}
+  {% if encrypted %}
+  ENCRYPTED
+  {% endif %}
+  {% if overwrite %}
+  ALLOWOVERWRITE
+  {% endif %}
+  {% if not parallel %}
+  PARALLEL OFF
+  {% endif %}
+
+{% endmacro %}