Skip to content

Commit

Permalink
Support domain types
Browse files Browse the repository at this point in the history 
Type domain types as their underlying type.

Closes singer-io#97.
  • Loading branch information
@Limess
Limess committed Jul 20, 2020
1 parent e990e88 commit ccb5740
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 3 deletions.
3 changes: 1 addition & 2 deletions tap_postgres/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,6 @@ def produce_table_info(conn):
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor, name='stitch_cursor') as cur:
cur.itersize = post_db.cursor_iter_size
table_info = {}
# SELECT CASE WHEN $2.typtype = 'd' THEN $2.typbasetype ELSE $1.atttypid END
cur.execute("""
SELECT
pg_class.reltuples::BIGINT AS approximate_row_count,
Expand All @@ -244,7 +243,7 @@ def produce_table_info(conn):
pg_class.relname AS table_name,
attname AS column_name,
i.indisprimary AS primary_key,
format_type(a.atttypid, NULL::integer) AS data_type,
format_type(CASE WHEN pgt.typtype = 'd' THEN pgt.typbasetype ELSE a.atttypid END, NULL::integer) AS data_type,
information_schema._pg_char_max_length(CASE WHEN COALESCE(subpgt.typtype, pgt.typtype) = 'd'
THEN COALESCE(subpgt.typbasetype, pgt.typbasetype) ELSE COALESCE(subpgt.oid, pgt.oid)
END,
Expand Down
8 changes: 8 additions & 0 deletions tests/test_postgres_full_table_replication.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
'our_inet': {'type': ['null', 'string']},
'our_mac': {'type': ['null', 'string']},
'our_alignment_enum': {'type': ['null', 'string']},
'our_text_domain': {'type': ['null', 'string']},
'our_money': {'type': ['null', 'string']}
}}}

Expand Down Expand Up @@ -123,6 +124,9 @@ def setUp(self):
cur.execute(""" DROP TYPE IF EXISTS ALIGNMENT CASCADE """)
cur.execute(""" CREATE TYPE ALIGNMENT AS ENUM ('good', 'bad', 'ugly') """)

cur.execute(""" DROP DOMAIN IF EXISTS text_domain_type CASCADE """)
cur.execute(""" CREATE DOMAIN text_domain_type text""")

create_table_sql = """
CREATE TABLE {} (id SERIAL PRIMARY KEY,
our_varchar VARCHAR,
Expand Down Expand Up @@ -150,6 +154,7 @@ def setUp(self):
our_cidr cidr,
our_mac macaddr,
our_alignment_enum ALIGNMENT,
our_text_domain domain_type,
our_money money)
""".format(canonicalized_table_name(test_schema_name, test_table_name, cur), NUMERIC_PRECISION, NUMERIC_SCALE)

Expand Down Expand Up @@ -188,6 +193,7 @@ def setUp(self):
'our_inet': '192.168.100.128/24',
'our_mac' : '08:00:2b:01:02:03',
'our_alignment_enum': 'good',
'our_text_domain': 'hello, world!',
'our_money': '100.1122',
}

Expand Down Expand Up @@ -338,6 +344,7 @@ def test_run(self):
('properties', 'our_cidr'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'cidr'},
('properties', 'our_mac'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'macaddr'},
('properties', 'our_alignment_enum'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'alignment'},
('properties', 'our_text_domain'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'text'},
('properties', 'our_money'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'money'}},
metadata.to_map(md))

Expand Down Expand Up @@ -414,6 +421,7 @@ def test_run(self):
'our_cidr' : self.rec_1['our_cidr'],
'our_mac' : self.rec_1['our_mac'],
'our_alignment_enum' : self.rec_1['our_alignment_enum'],
'our_text_domain' : self.rec_1['our_text_domain'],
'our_money' : '$100.11'
}

Expand Down
9 changes: 8 additions & 1 deletion tests/test_postgres_logical_replication.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
'our_inet': {'type': ['null', 'string']},
'our_mac': {'type': ['null', 'string']},
'our_alignment_enum': {'type': ['null', 'string']},
'our_text_domain': {'type': ['null', 'string']},
'our_money': {'type': ['null', 'string']}}}}


Expand Down Expand Up @@ -139,6 +140,8 @@ def setUp(self):
cur.execute(""" DROP TYPE IF EXISTS ALIGNMENT CASCADE """)
cur.execute(""" CREATE TYPE ALIGNMENT AS ENUM ('good', 'bad', 'ugly') """)

cur.execute(""" DROP DOMAIN IF EXISTS text_domain_type CASCADE """)
cur.execute(""" CREATE DOMAIN text_domain_type text""")

create_table_sql = """
CREATE TABLE {} (id SERIAL PRIMARY KEY,
Expand Down Expand Up @@ -168,6 +171,7 @@ def setUp(self):
our_inet inet,
our_mac macaddr,
our_alignment_enum ALIGNMENT,
our_text_domain text_domain_type,
our_money money)
""".format(canonicalized_table_name(test_schema_name, test_table_name, cur))

Expand Down Expand Up @@ -206,7 +210,8 @@ def setUp(self):
'our_cidr' : '192.168.100.128/25',
'our_inet': '192.168.100.128/24',
'our_mac' : '08:00:2b:01:02:03',
'our_alignment_enum': 'bad'}
'our_alignment_enum': 'bad',
'our_text_domain': 'hello, world!'}


insert_record(cur, test_table_name, self.rec_1)
Expand Down Expand Up @@ -472,6 +477,7 @@ def test_run(self):
'our_inet': '192.168.102.128',
'our_mac': self.rec_3['our_mac'],
'our_alignment_enum' : None,
'our_text_domain': None,
'our_money' :'$412.12'
}
self.assertEqual(set(actual_record_1.keys()), set(expected_inserted_record.keys()),
Expand Down Expand Up @@ -628,6 +634,7 @@ def test_run(self):
'our_inet': self.rec_1['our_inet'],
'our_mac': self.rec_1['our_mac'],
'our_alignment_enum' : 'bad',
'our_text_domain': 'hello, world!',
'our_money' : '$56.81'
}

Expand Down
87 changes: 87 additions & 0 deletions tests/unittests/test_discovery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,94 @@ def test_catalog(self):
'type': 'object',
'definitions' : tap_postgres.BASE_RECURSIVE_SCHEMAS},
stream_dict.get('schema'))
class TestDomainsTable(unittest.TestCase):
maxDiff = None
table_name = "CHICKEN TIMES"

def setUp(self):
table_spec = {
"columns": [
{
"name": "our_test_domain_pk",
"type": "test_domain",
"primary_key": True,
},
{"name": "our_test_domain", "type": "test_domain"},
{"name": "our_test_integer_domain", "type": "test_integer_domain"},
],
"name": TestHStoreTable.table_name,
}
with get_test_connection() as conn:
cur = conn.cursor()
cur.execute(""" DROP DOMAIN IF EXISTS test_domain CASCADE """)
cur.execute(""" CREATE DOMAIN test_domain AS text; """)
cur.execute(""" DROP DOMAIN IF EXISTS test_integer_domain CASCADE """)
cur.execute(""" CREATE DOMAIN test_integer_domain AS integer; """)

ensure_test_table(table_spec)

def test_catalog(self):
conn_config = get_test_connection_config()
streams = tap_postgres.do_discovery(conn_config)
chicken_streams = [
s for s in streams if s["tap_stream_id"] == "public-CHICKEN TIMES"
]
self.assertEqual(len(chicken_streams), 1)
stream_dict = chicken_streams[0]
stream_dict.get("metadata").sort(key=lambda md: md["breadcrumb"])

with get_test_connection() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(
"""INSERT INTO "CHICKEN TIMES" (our_test_domain_pk, our_test_domain, our_test_integer_domain) VALUES ('sad', 'happy', 3)"""
)
cur.execute("""SELECT * FROM "CHICKEN TIMES" """)

self.assertEqual(
metadata.to_map(stream_dict.get("metadata")),
{
(): {
"table-key-properties": ["our_test_domain_pk"],
"database-name": "postgres",
"schema-name": "public",
"is-view": False,
"row-count": 0,
},
("properties", "our_test_domain_pk"): {
"inclusion": "automatic",
"sql-datatype": "text",
"selected-by-default": True,
},
("properties", "our_test_domain"): {
"inclusion": "available",
"sql-datatype": "text",
"selected-by-default": True,
},
("properties", "our_test_integer_domain"): {
"inclusion": "available",
"sql-datatype": "integer",
"selected-by-default": True,
},
},
)

self.assertEqual(
{
"properties": {
"our_test_domain_pk": {"type": ["string"]},
"our_test_domain": {"type": ["null", "string"]},
"our_test_integer_domain": {
"minimum": -2147483648,
"maximum": 2147483647,
"type": ["null", "integer"],
},
},
"type": "object",
"definitions": BASE_RECURSIVE_SCHEMAS,
},
stream_dict.get("schema"),
)

class TestArraysTable(unittest.TestCase):
maxDiff = None
table_name = 'CHICKEN TIMES'
Expand Down

0 comments on commit ccb5740

Please sign in to comment.