Skip to content

Commit

Permalink
test: Test with singer-sdk @ main
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon committed Nov 28, 2024
1 parent 7e7d194 commit 2efa25b
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 31 deletions.
32 changes: 18 additions & 14 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ sqlalchemy = "2.0.36"
sshtunnel = "0.4.0"

[tool.poetry.dependencies.singer-sdk]
version = "~=0.42.1"
git = "https://github.com/meltano/sdk.git"
branch = "edgarrmondragon/refactor/sqltojsonschema-fromconfig"
extras = ["faker"]

[tool.poetry.group.dev.dependencies]
Expand All @@ -56,7 +57,8 @@ types-jsonschema = ">=4.19.0.3"
types-psycopg2 = ">=2.9.21.20240118"

[tool.poetry.dev-dependencies.singer-sdk]
version = "*"
git = "https://github.com/meltano/sdk.git"
branch = "edgarrmondragon/refactor/sqltojsonschema-fromconfig"
extras = ["testing"]

[tool.mypy]
Expand Down
39 changes: 24 additions & 15 deletions tap_postgres/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,36 @@
from collections.abc import Iterable, Mapping

from singer_sdk.helpers.types import Context
from sqlalchemy.dialects import postgresql
from sqlalchemy.engine import Engine
from sqlalchemy.engine.reflection import Inspector


class PostgresSQLToJSONSchema(SQLToJSONSchema):
"""Custom SQL to JSON Schema conversion for Postgres."""

def __init__(self, dates_as_string: bool, json_as_object: bool, *args, **kwargs):
def __init__(self, *, dates_as_string: bool, json_as_object: bool, **kwargs):
"""Initialize the SQL to JSON Schema converter."""
super().__init__(*args, **kwargs)
super().__init__(**kwargs)
self.dates_as_string = dates_as_string
self.json_as_object = json_as_object

@SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined]
@classmethod
def from_config(cls, config: dict) -> PostgresSQLToJSONSchema:
"""Instantiate the SQL to JSON Schema converter from a config dictionary."""
return cls(
dates_as_string=config["dates_as_string"],
json_as_object=config["json_as_object"],
)

@functools.singledispatchmethod
def to_jsonschema(self, column_type: Any) -> dict:
"""Override the default mapping for NUMERIC columns.
For example, a scale of 4 translates to a multipleOf 0.0001.
"""
return super().to_jsonschema(column_type)

@to_jsonschema.register # type: ignore[attr-defined]
def array_to_jsonschema(self, column_type: postgresql.ARRAY) -> dict:
"""Override the default mapping for NUMERIC columns.
Expand All @@ -55,21 +70,21 @@ def array_to_jsonschema(self, column_type: postgresql.ARRAY) -> dict:
"items": self.to_jsonschema(column_type.item_type),
}

@SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined]
@to_jsonschema.register # type: ignore[attr-defined]
def json_to_jsonschema(self, column_type: postgresql.JSON) -> dict:
"""Override the default mapping for JSON and JSONB columns."""
if self.json_as_object:
return {"type": ["object", "null"]}
return {"type": ["string", "number", "integer", "array", "object", "boolean"]}

@SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined]
@to_jsonschema.register # type: ignore[attr-defined]
def datetime_to_jsonschema(self, column_type: sqlalchemy.types.DateTime) -> dict:
"""Override the default mapping for DATETIME columns."""
if self.dates_as_string:
return {"type": ["string", "null"]}
return super().datetime_to_jsonschema(column_type)

@SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined]
@to_jsonschema.register # type: ignore[attr-defined]
def date_to_jsonschema(self, column_type: sqlalchemy.types.Date) -> dict:
"""Override the default mapping for DATE columns."""
if self.dates_as_string:
Expand Down Expand Up @@ -133,6 +148,8 @@ def patched_conform(
class PostgresConnector(SQLConnector):
"""Connects to the Postgres SQL source."""

sql_to_jsonschema_converter = PostgresSQLToJSONSchema

def __init__(
self,
config: dict | None = None,
Expand Down Expand Up @@ -161,14 +178,6 @@ def __init__(

super().__init__(config=config, sqlalchemy_url=sqlalchemy_url)

@functools.cached_property
def sql_to_jsonschema(self):
"""Return a mapping of SQL types to JSON Schema types."""
return PostgresSQLToJSONSchema(
dates_as_string=self.config["dates_as_string"],
json_as_object=self.config["json_as_object"],
)

def get_schema_names(self, engine: Engine, inspected: Inspector) -> list[str]:
"""Return a list of schema names in DB, or overrides with user-provided values.
Expand Down

0 comments on commit 2efa25b

Please sign in to comment.