Skip to content

Commit

Permalink
Fix additional errors
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon committed Jan 16, 2024
1 parent 0238b82 commit 9329a43
Show file tree
Hide file tree
Showing 7 changed files with 35 additions and 35 deletions.
19 changes: 6 additions & 13 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,32 +1,25 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
rev: v4.5.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 22.10.0
rev: 23.12.1
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.12.0
rev: 5.13.2
hooks:
- id: isort
- repo: https://github.com/python-poetry/poetry
rev: 1.3.2
rev: 1.7.0
hooks:
- id: poetry-check
- id: poetry-lock
args: [--no-update]
- repo: https://github.com/pre-commit/mirrors-mypy
rev: 'v0.991'
hooks:
- id: mypy
exclude: tests
additional_dependencies:
- types-paramiko
- repo: https://github.com/pycqa/flake8
rev: 6.1.0
rev: 7.0.0
hooks:
- id: flake8
- id: flake8
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -261,4 +261,4 @@ Note also that using log-based replication will cause the replication key for al
"*":
replication_method: LOG_BASED
replication_key: _sdc_lsn
```
```
2 changes: 1 addition & 1 deletion log_based/init.sql
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');
SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');
2 changes: 1 addition & 1 deletion poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 16 additions & 12 deletions tap_postgres/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ def sdk_typing_object(
| th.DateType
| th.StringType
| th.BooleanType
| th.CustomType
):
"""Return the JSON Schema dict that describes the sql type.
Expand Down Expand Up @@ -209,14 +210,15 @@ def sdk_typing_object(
| th.IntegerType
| th.DateType
| th.StringType
| th.BooleanType,
| th.BooleanType
| th.CustomType,
] = {
"jsonb": th.CustomType({
"type": ["string", "number", "integer", "array", "object", "boolean"]
}),
"json": th.CustomType({
"type": ["string", "number", "integer", "array", "object", "boolean"]
}),
"jsonb": th.CustomType(
{"type": ["string", "number", "integer", "array", "object", "boolean"]}
),
"json": th.CustomType(
{"type": ["string", "number", "integer", "array", "object", "boolean"]}
),
"timestamp": th.DateTimeType(),
"datetime": th.DateTimeType(),
"date": th.DateType(),
Expand Down Expand Up @@ -475,11 +477,13 @@ def consume(self, message) -> dict | None:
elif message_payload["action"] in delete_actions:
for column in message_payload["identity"]:
row.update({column["name"]: column["value"]})
row.update({
"_sdc_deleted_at": datetime.datetime.utcnow().strftime(
r"%Y-%m-%dT%H:%M:%SZ"
)
})
row.update(
{
"_sdc_deleted_at": datetime.datetime.utcnow().strftime(
r"%Y-%m-%dT%H:%M:%SZ"
)
}
)
row.update({"_sdc_lsn": message.data_start})
elif message_payload["action"] in truncate_actions:
self.logger.debug(
Expand Down
6 changes: 3 additions & 3 deletions tests/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@
from singer_sdk.testing.runners import TapTestRunner
from sqlalchemy import Column, DateTime, Integer, MetaData, Numeric, String, Table, text
from sqlalchemy.dialects.postgresql import (
ARRAY,
BIGINT,
DATE,
JSON,
JSONB,
TIME,
TIMESTAMP,
ARRAY,
)
from tests.settings import DB_SCHEMA_NAME, DB_SQLALCHEMY_URL
from tests.test_replication_key import TABLE_NAME, TapTestReplicationKey
Expand Down Expand Up @@ -418,13 +418,13 @@ def run_sync_dry_run(self) -> bool:
return True


def test_invalid_python_dates():
def test_invalid_python_dates(): # noqa: C901
"""Some dates are invalid in python, but valid in Postgres
Check out https://www.psycopg.org/psycopg3/docs/advanced/adapt.html#example-handling-infinity-date
for more information.
"""
""" # noqa: E501
table_name = "test_invalid_python_dates"
engine = sqlalchemy.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True)

Expand Down
11 changes: 7 additions & 4 deletions tests/test_log_based.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
import sqlalchemy
from sqlalchemy import Column, MetaData, Table
from sqlalchemy.dialects.postgresql import BIGINT, TEXT
from tap_postgres.tap import TapPostgres
from tests.test_core import PostgresTestRunner

from tap_postgres.tap import TapPostgres

LOG_BASED_CONFIG = {
"host": "localhost",
Expand All @@ -15,6 +15,7 @@
"database": "postgres",
}


def test_null_append():
"""LOG_BASED syncs failed with string property types. (issue #294).
Expand All @@ -23,14 +24,16 @@ def test_null_append():
LOG_BASED replication can still append the "null" option to a property's type.
"""
table_name = "test_null_append"
engine = sqlalchemy.create_engine("postgresql://postgres:postgres@localhost:5434/postgres")
engine = sqlalchemy.create_engine(
"postgresql://postgres:postgres@localhost:5434/postgres"
)

metadata_obj = MetaData()
table = Table(
table_name,
metadata_obj,
Column("id", BIGINT, primary_key = True),
Column("data", TEXT, nullable = True)
Column("id", BIGINT, primary_key=True),
Column("data", TEXT, nullable=True),
)
with engine.connect() as conn:
table.drop(conn, checkfirst=True)
Expand Down

0 comments on commit 9329a43

Please sign in to comment.