From 911781d6ccbff4e36e8589fedb1d1053e03e7419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Wed, 10 Jan 2024 17:04:33 -0600 Subject: [PATCH] Fix additional errors --- .pre-commit-config.yaml | 19 ++++++------------- README.md | 2 +- log_based/init.sql | 2 +- poetry.lock | 2 +- tap_postgres/client.py | 28 ++++++++++++++++------------ tests/test_core.py | 6 +++--- tests/test_log_based.py | 11 +++++++---- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cdf38d9..f64ed824 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,32 +1,25 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 23.12.1 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/python-poetry/poetry - rev: 1.3.2 + rev: 1.7.0 hooks: - id: poetry-check - id: poetry-lock args: [--no-update] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.991' - hooks: - - id: mypy - exclude: tests - additional_dependencies: - - types-paramiko - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - - id: flake8 \ No newline at end of file + - id: flake8 diff --git a/README.md b/README.md index 7f538dc1..226d317e 100644 --- a/README.md +++ b/README.md @@ -261,4 +261,4 @@ Note also that using log-based replication will cause the replication key for al "*": replication_method: LOG_BASED replication_key: _sdc_lsn - ``` \ No newline at end of file + ``` diff --git a/log_based/init.sql b/log_based/init.sql index 4cd87906..a700552f 100644 --- a/log_based/init.sql +++ b/log_based/init.sql @@ -1 +1 @@ -SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json'); \ No newline at end of file +SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json'); diff --git a/poetry.lock b/poetry.lock index daa03c58..f343ceb5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. [[package]] name = "appdirs" diff --git a/tap_postgres/client.py b/tap_postgres/client.py index 4fdabd3e..141c6454 100644 --- a/tap_postgres/client.py +++ b/tap_postgres/client.py @@ -178,6 +178,7 @@ def sdk_typing_object( | th.DateType | th.StringType | th.BooleanType + | th.CustomType ): """Return the JSON Schema dict that describes the sql type. @@ -209,14 +210,15 @@ def sdk_typing_object( | th.IntegerType | th.DateType | th.StringType - | th.BooleanType, + | th.BooleanType + | th.CustomType, ] = { - "jsonb": th.CustomType({ - "type": ["string", "number", "integer", "array", "object", "boolean"] - }), - "json": th.CustomType({ - "type": ["string", "number", "integer", "array", "object", "boolean"] - }), + "jsonb": th.CustomType( + {"type": ["string", "number", "integer", "array", "object", "boolean"]} + ), + "json": th.CustomType( + {"type": ["string", "number", "integer", "array", "object", "boolean"]} + ), "timestamp": th.DateTimeType(), "datetime": th.DateTimeType(), "date": th.DateType(), @@ -475,11 +477,13 @@ def consume(self, message) -> dict | None: elif message_payload["action"] in delete_actions: for column in message_payload["identity"]: row.update({column["name"]: column["value"]}) - row.update({ - "_sdc_deleted_at": datetime.datetime.utcnow().strftime( - r"%Y-%m-%dT%H:%M:%SZ" - ) - }) + row.update( + { + "_sdc_deleted_at": datetime.datetime.utcnow().strftime( + r"%Y-%m-%dT%H:%M:%SZ" + ) + } + ) row.update({"_sdc_lsn": message.data_start}) elif message_payload["action"] in truncate_actions: self.logger.debug( diff --git a/tests/test_core.py b/tests/test_core.py index 1bb8614d..9736ca80 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -11,13 +11,13 @@ from singer_sdk.testing.runners import TapTestRunner from sqlalchemy import Column, DateTime, Integer, MetaData, Numeric, String, Table, text from sqlalchemy.dialects.postgresql import ( + ARRAY, BIGINT, DATE, JSON, JSONB, TIME, TIMESTAMP, - ARRAY, ) from tests.settings import DB_SCHEMA_NAME, DB_SQLALCHEMY_URL from tests.test_replication_key import TABLE_NAME, TapTestReplicationKey @@ -418,13 +418,13 @@ def run_sync_dry_run(self) -> bool: return True -def test_invalid_python_dates(): +def test_invalid_python_dates(): # noqa: C901 """Some dates are invalid in python, but valid in Postgres Check out https://www.psycopg.org/psycopg3/docs/advanced/adapt.html#example-handling-infinity-date for more information. - """ + """ # noqa: E501 table_name = "test_invalid_python_dates" engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True) diff --git a/tests/test_log_based.py b/tests/test_log_based.py index 0048c0a4..889ee12b 100644 --- a/tests/test_log_based.py +++ b/tests/test_log_based.py @@ -3,9 +3,9 @@ import sqlalchemy from sqlalchemy import Column, MetaData, Table from sqlalchemy.dialects.postgresql import BIGINT, TEXT -from tap_postgres.tap import TapPostgres from tests.test_core import PostgresTestRunner +from tap_postgres.tap import TapPostgres LOG_BASED_CONFIG = { "host": "localhost", @@ -15,6 +15,7 @@ "database": "postgres", } + def test_null_append(): """LOG_BASED syncs failed with string property types. (issue #294). @@ -23,14 +24,16 @@ def test_null_append(): LOG_BASED replication can still append the "null" option to a property's type. """ table_name = "test_null_append" - engine = sqlalchemy.create_engine("postgresql://postgres:postgres@localhost:5434/postgres") + engine = sqlalchemy.create_engine( + "postgresql://postgres:postgres@localhost:5434/postgres" + ) metadata_obj = MetaData() table = Table( table_name, metadata_obj, - Column("id", BIGINT, primary_key = True), - Column("data", TEXT, nullable = True) + Column("id", BIGINT, primary_key=True), + Column("data", TEXT, nullable=True), ) with engine.connect() as conn: table.drop(conn, checkfirst=True)