Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Custom Iceberg base_location_root #1289

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
+163 −3
Open

Feature: Custom Iceberg base_location_root #1289

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
6360ea0
update relation & add tests
LProcopi15 Jan 13, 2025
1c2a14a
switch if/else pattern
LProcopi15 Jan 13, 2025
76862d6
add change log
LProcopi15 Jan 13, 2025
774f5ad
add test for dynamic table with path and subpath
LProcopi15 Jan 14, 2025
016ccf8
modify base_location config
LProcopi15 Jan 15, 2025
c4ffbae
update to base_location_root, include schema and table name in path
LProcopi15 Jan 16, 2025
4575ffe
resolve linting error (f-string nested double quotes)
LProcopi15 Jan 16, 2025
087d7da
resolve linting error (move to new line)
LProcopi15 Jan 16, 2025
17b4821
add unit tests & make final edits to functional tests
LProcopi15 Jan 17, 2025
4f57390
lint unit test
LProcopi15 Jan 17, 2025
8d34e8f
update changie
LProcopi15 Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20250113-133414.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Added support for custom iceberg base_location
time: 2025-01-13T13:34:14.326047-08:00
custom:
Author: LProcopi15
Issue: "1284"
1 change: 1 addition & 0 deletions dbt/adapters/snowflake/impl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ class SnowflakeConfig(AdapterConfig):
# extended formats
table_format: Optional[str] = None
external_volume: Optional[str] = None
base_location: Optional[str] = None
base_location_subpath: Optional[str] = None


Expand Down
4 changes: 4 additions & 0 deletions dbt/adapters/snowflake/relation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,10 @@ def get_ddl_prefix_for_alter(self) -> str:
def get_iceberg_ddl_options(self, config: RelationConfig) -> str:
base_location: str = f"_dbt/{self.schema}/{self.name}"

# If the base_location config is supplied, overwrite the default value
if base_path := config.get("base_location"):
base_location = f"{base_path}/{self.name}"

if subpath := config.get("base_location_subpath"):
base_location += f"/{subpath}"

Expand Down
63 changes: 63 additions & 0 deletions tests/functional/iceberg/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,37 @@
select * from {{ ref('first_table') }}
"""

_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH = """
{{
config(
transient = "true",
materialized = "table",
cluster_by=['id'],
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location="base_path",
Copy link

@amychen1776 amychen1776 Jan 15, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for this test, do we not need an actual base_path? (I honestly have no idea but wanted to ask)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the path doesn't already exist, it is created by Snowflake when you run create or replace iceberg table, so for our test we don't need to use an existing path

)
}}
select * from {{ ref('first_table') }}
"""

_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH = """
{{
config(
transient = "true",
materialized = "table",
cluster_by=['id'],
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location="base_path",
base_location_subpath="subpath",
)
}}
select * from {{ ref('first_table') }}
"""

_MODEL_BASIC_DYNAMIC_TABLE_MODEL = """
{{ config(
materialized='dynamic_table',
Expand All @@ -36,6 +67,38 @@
select * from {{ ref('first_table') }}
"""

_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH = """
{{
config(
transient = "true",
materialized = "table",
cluster_by=['id'],
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location="base_path",
)
}}
select * from {{ ref('first_table') }}
"""

_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH = """
{{
config(
transient = "true",
materialized = "table",
cluster_by=['id'],
table_format="iceberg",
external_volume="s3_iceberg_snow",
base_location="base_path",
base_location_subpath='subpath',
)
}}
select * from {{ ref('first_table') }}
"""


_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH = """
{{ config(
materialized='dynamic_table',
Expand Down
12 changes: 10 additions & 2 deletions tests/functional/iceberg/test_table_basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
from tests.functional.iceberg.models import (
_MODEL_BASIC_TABLE_MODEL,
_MODEL_BASIC_ICEBERG_MODEL,
_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH,
_MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH,
_MODEL_BASIC_DYNAMIC_TABLE_MODEL,
_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH,
_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH,
_MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH,
_MODEL_BUILT_ON_ICEBERG_TABLE,
_MODEL_TABLE_BEFORE_SWAP,
Expand All @@ -26,14 +30,18 @@ def models(self):
return {
"first_table.sql": _MODEL_BASIC_TABLE_MODEL,
"iceberg_table.sql": _MODEL_BASIC_ICEBERG_MODEL,
"iceberg_tableb.sql": _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH,
"iceberg_tablec.sql": _MODEL_BASIC_ICEBERG_MODEL_WITH_PATH_SUBPATH,
"table_built_on_iceberg_table.sql": _MODEL_BUILT_ON_ICEBERG_TABLE,
"dynamic_table.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL,
"dynamic_tableb.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH,
"dynamic_tableb.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH,
"dynamic_tablec.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_PATH_SUBPATH,
"dynamic_tabled.sql": _MODEL_BASIC_DYNAMIC_TABLE_MODEL_WITH_SUBPATH,
}

def test_iceberg_tables_build_and_can_be_referred(self, project):
run_results = run_dbt()
assert len(run_results) == 5
assert len(run_results) == 9


class TestIcebergTableTypeBuildsOnExistingTable:
Expand Down
Loading