From 9df942d89e8b97bfd0757f7be37b0b6b42e1f3ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:06:51 -0600 Subject: [PATCH 1/3] test: Use `sqlalchemy.text` in tests (#305) --- tests/test_core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 37ef9aa7..306316bb 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -9,7 +9,7 @@ from faker import Faker from singer_sdk.testing import get_tap_test_class, suites from singer_sdk.testing.runners import TapTestRunner -from sqlalchemy import Column, DateTime, Integer, MetaData, Numeric, String, Table +from sqlalchemy import Column, DateTime, Integer, MetaData, Numeric, String, Table, text from sqlalchemy.dialects.postgresql import BIGINT, DATE, JSON, JSONB, TIME, TIMESTAMP from test_replication_key import TABLE_NAME, TapTestReplicationKey from test_selected_columns_only import ( @@ -51,7 +51,7 @@ def setup_test_table(table_name, sqlalchemy_url): ) with engine.connect() as conn: metadata_obj.create_all(conn) - conn.execute(f"TRUNCATE TABLE {table_name}") + conn.execute(text(f"TRUNCATE TABLE {table_name}")) for _ in range(1000): insert = test_replication_key_table.insert().values( updated_at=fake.date_between(date1, date2), name=fake.name() @@ -62,7 +62,7 @@ def setup_test_table(table_name, sqlalchemy_url): def teardown_test_table(table_name, sqlalchemy_url): engine = sqlalchemy.create_engine(sqlalchemy_url) with engine.connect() as conn: - conn.execute(f"DROP TABLE {table_name}") + conn.execute(text(f"DROP TABLE {table_name}")) custom_test_replication_key = suites.TestSuite( @@ -300,7 +300,7 @@ def test_filter_schemas(): table = Table(table_name, metadata_obj, Column("id", BIGINT), schema="new_schema") with engine.connect() as conn: - conn.execute("CREATE SCHEMA IF NOT EXISTS new_schema") + conn.execute(text("CREATE SCHEMA IF NOT EXISTS new_schema")) if table.exists(conn): table.drop(conn) metadata_obj.create_all(conn) From 40ff4cf302d48ba9be929740f143db5a36f14381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Thu, 7 Dec 2023 19:25:05 -0600 Subject: [PATCH 2/3] test: Use `future=True` param to create engines (#306) --- tests/test_core.py | 28 ++++++++++++++-------------- tests/test_replication_key.py | 9 +++++---- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 306316bb..50f470de 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -36,7 +36,7 @@ def setup_test_table(table_name, sqlalchemy_url): """setup any state specific to the execution of the given module.""" - engine = sqlalchemy.create_engine(sqlalchemy_url) + engine = sqlalchemy.create_engine(sqlalchemy_url, future=True) fake = Faker() date1 = datetime.date(2022, 11, 1) @@ -49,7 +49,7 @@ def setup_test_table(table_name, sqlalchemy_url): Column("updated_at", DateTime(), nullable=False), Column("name", String()), ) - with engine.connect() as conn: + with engine.begin() as conn: metadata_obj.create_all(conn) conn.execute(text(f"TRUNCATE TABLE {table_name}")) for _ in range(1000): @@ -60,8 +60,8 @@ def setup_test_table(table_name, sqlalchemy_url): def teardown_test_table(table_name, sqlalchemy_url): - engine = sqlalchemy.create_engine(sqlalchemy_url) - with engine.connect() as conn: + engine = sqlalchemy.create_engine(sqlalchemy_url, future=True) + with engine.begin() as conn: conn.execute(text(f"DROP TABLE {table_name}")) @@ -137,7 +137,7 @@ def test_temporal_datatypes(): schema checks, and performs similar tests on times and timestamps. """ table_name = "test_temporal_datatypes" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"]) + engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) metadata_obj = MetaData() table = Table( @@ -147,7 +147,7 @@ def test_temporal_datatypes(): Column("column_time", TIME), Column("column_timestamp", TIMESTAMP), ) - with engine.connect() as conn: + with engine.begin() as conn: if table.exists(conn): table.drop(conn) metadata_obj.create_all(conn) @@ -197,7 +197,7 @@ def test_temporal_datatypes(): def test_jsonb_json(): """JSONB and JSON Objects weren't being selected, make sure they are now""" table_name = "test_jsonb_json" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"]) + engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) metadata_obj = MetaData() table = Table( @@ -206,7 +206,7 @@ def test_jsonb_json(): Column("column_jsonb", JSONB), Column("column_json", JSON), ) - with engine.connect() as conn: + with engine.begin() as conn: if table.exists(conn): table.drop(conn) metadata_obj.create_all(conn) @@ -248,7 +248,7 @@ def test_jsonb_json(): def test_decimal(): """Schema was wrong for Decimal objects. Check they are correctly selected.""" table_name = "test_decimal" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"]) + engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) metadata_obj = MetaData() table = Table( @@ -256,7 +256,7 @@ def test_decimal(): metadata_obj, Column("column", Numeric()), ) - with engine.connect() as conn: + with engine.begin() as conn: if table.exists(conn): table.drop(conn) metadata_obj.create_all(conn) @@ -294,12 +294,12 @@ def test_decimal(): def test_filter_schemas(): """Only return tables from a given schema""" table_name = "test_filter_schemas" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"]) + engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) metadata_obj = MetaData() table = Table(table_name, metadata_obj, Column("id", BIGINT), schema="new_schema") - with engine.connect() as conn: + with engine.begin() as conn: conn.execute(text("CREATE SCHEMA IF NOT EXISTS new_schema")) if table.exists(conn): table.drop(conn) @@ -333,7 +333,7 @@ def test_invalid_python_dates(): """ table_name = "test_invalid_python_dates" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"]) + engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) metadata_obj = MetaData() table = Table( @@ -342,7 +342,7 @@ def test_invalid_python_dates(): Column("date", DATE), Column("datetime", DateTime), ) - with engine.connect() as conn: + with engine.begin() as conn: if table.exists(conn): table.drop(conn) metadata_obj.create_all(conn) diff --git a/tests/test_replication_key.py b/tests/test_replication_key.py index 30f3f769..d64f903d 100644 --- a/tests/test_replication_key.py +++ b/tests/test_replication_key.py @@ -1,4 +1,5 @@ """Tests standard tap features using the built-in SDK tests library.""" + import copy import json @@ -54,7 +55,7 @@ def test_null_replication_key_with_start_date(): greater than the start date should be synced. """ table_name = "test_null_replication_key_with_start_date" - engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"]) + engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) metadata_obj = MetaData() table = Table( @@ -63,7 +64,7 @@ def test_null_replication_key_with_start_date(): Column("data", String()), Column("updated_at", TIMESTAMP), ) - with engine.connect() as conn: + with engine.begin() as conn: if table.exists(conn): table.drop(conn) metadata_obj.create_all(conn) @@ -111,7 +112,7 @@ def test_null_replication_key_without_start_date(): modified_config = copy.deepcopy(SAMPLE_CONFIG) modified_config["start_date"] = None - engine = sqlalchemy.create_engine(modified_config["sqlalchemy_url"]) + engine = sqlalchemy.create_engine(modified_config["sqlalchemy_url"], future=True) metadata_obj = MetaData() table = Table( @@ -120,7 +121,7 @@ def test_null_replication_key_without_start_date(): Column("data", String()), Column("updated_at", TIMESTAMP), ) - with engine.connect() as conn: + with engine.begin() as conn: if table.exists(conn): table.drop(conn) metadata_obj.create_all(conn) From 678737e0f097854d9d49b499a641b74d0d7df457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:54:51 -0600 Subject: [PATCH 3/3] chore: Add pendulum as an explicit dev dependency (#308) So we avoid importing a transitive dependency! --- poetry.lock | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index aa3d9f17..fa6fd90c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1799,4 +1799,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "<3.12,>=3.8.1" -content-hash = "642a5113757608430c49f918dc414ab70b0c4d19dfb312f4bcf70d1ad4274939" +content-hash = "8e48dbe273853c675d6fa6b194e8c82966d7dd9b1948fc4e3e39b2c64365f758" diff --git a/pyproject.toml b/pyproject.toml index 50d0b8a5..c21fb253 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ faker = ">=18.5.1,<21.0.0" flake8 = "^6.0.0" isort = "^5.10.1" mypy = "1.7.1" +pendulum = "~=2.1" pre-commit = "^3.0.4" pydocstyle = "^6.1.1" singer-sdk = {version = "*", extras = ["testing"]}