diff --git a/poetry.lock b/poetry.lock index 0aef3f58..7a9f8587 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "appdirs" @@ -1554,36 +1554,34 @@ files = [ [[package]] name = "singer-sdk" -version = "0.42.1" +version = "0.43.0a2.post4.dev0+2f29bfc4" description = "A framework for building Singer taps" optional = false -python-versions = ">=3.8" -files = [ - {file = "singer_sdk-0.42.1-py3-none-any.whl", hash = "sha256:cc588927ff87b7aea0087c57c37c85bbe95f9b7c52ebb265abb50a184cec6ad8"}, - {file = "singer_sdk-0.42.1.tar.gz", hash = "sha256:92019c3f2e476f5c17eaa381e35d3c0ff428814587f8424149a4b28904f75ca0"}, -] +python-versions = ">=3.9" +files = [] +develop = false [package.dependencies] backoff = {version = ">=2.0.0", markers = "python_version < \"4\""} backports-datetime-fromisoformat = {version = ">=2.0.1", markers = "python_version < \"3.11\""} -click = ">=8.0,<9.0" -faker = {version = ">=22.5", optional = true, markers = "extra == \"faker\""} +click = "~=8.0" +faker = {version = ">=22.5", optional = true} fs = ">=2.4.16" fsspec = ">=2024.9.0" importlib-metadata = {version = "<9.0.0", markers = "python_version < \"3.12\""} -importlib-resources = {version = ">=5.12.0,<6.2.0 || >6.2.0,<6.3.0 || >6.3.0,<6.3.1 || >6.3.1", markers = "python_version < \"3.10\""} +importlib-resources = {version = ">=5.12.0,!=6.2.0,!=6.3.0,!=6.3.1", markers = "python_version < \"3.10\""} inflection = ">=0.5.1" joblib = ">=1.3.0" jsonpath-ng = ">=1.5.3" jsonschema = ">=4.16.0" packaging = ">=23.1" -pytest = {version = ">=7.2.1", optional = true, markers = "extra == \"docs\" or extra == \"testing\""} +pytest = {version = ">=7.2.1", optional = true} python-dotenv = ">=0.20" PyYAML = ">=6.0" referencing = ">=0.30.0" requests = ">=2.25.1" setuptools = "<=70.3.0" -simpleeval = {version = ">=0.9.13,<1.0.1 || >1.0.1", markers = "python_version >= \"3.9\""} +simpleeval = ">=0.9.13,!=1.0.1" simplejson = ">=3.17.6" sqlalchemy = ">=1.4,<3.0" typing-extensions = ">=4.5.0" @@ -1592,11 +1590,17 @@ typing-extensions = ">=4.5.0" docs = ["furo (>=2024.5.6)", "myst-parser (>=3)", "pytest (>=7.2.1)", "sphinx (>=7)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-notfound-page (>=1.0.0)", "sphinx-reredirects (>=0.1.5)"] faker = ["faker (>=22.5)"] jwt = ["PyJWT (>=2.4,<3.0)", "cryptography (>=3.4.6)"] -parquet = ["numpy (>=1.22)", "numpy (>=1.22,<1.25)", "numpy (>=1.22,<2.1)", "pyarrow (>=13)", "pyarrow (>=13,<18)"] +parquet = ["numpy (>=1.22)", "numpy (>=1.22,<2.1)", "pyarrow (>=13)"] s3 = ["fs-s3fs (>=1.1.1)", "s3fs (>=2024.9.0)"] ssh = ["paramiko (>=3.3.0)"] testing = ["pytest (>=7.2.1)"] +[package.source] +type = "git" +url = "https://github.com/meltano/sdk.git" +reference = "edgarrmondragon/refactor/sqltojsonschema-fromconfig" +resolved_reference = "2f29bfc40b87c92411ea5b4872cd618d7d0ca87c" + [[package]] name = "six" version = "1.16.0" @@ -1901,4 +1905,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9" -content-hash = "1acb4ab3b0dc1229b9b75372766616224a5f02357283f670796d0c694720cd70" +content-hash = "6cbba3cb330e321b607497f82cbef6da3dfd4e8633897d8ec5e5f8741bc41cf9" diff --git a/pyproject.toml b/pyproject.toml index cca1bbd5..32afaf4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,8 @@ sqlalchemy = "2.0.36" sshtunnel = "0.4.0" [tool.poetry.dependencies.singer-sdk] -version = "~=0.42.1" +git = "https://github.com/meltano/sdk.git" +branch = "edgarrmondragon/refactor/sqltojsonschema-fromconfig" extras = ["faker"] [tool.poetry.group.dev.dependencies] @@ -56,7 +57,8 @@ types-jsonschema = ">=4.19.0.3" types-psycopg2 = ">=2.9.21.20240118" [tool.poetry.dev-dependencies.singer-sdk] -version = "*" +git = "https://github.com/meltano/sdk.git" +branch = "edgarrmondragon/refactor/sqltojsonschema-fromconfig" extras = ["testing"] [tool.mypy] diff --git a/tap_postgres/client.py b/tap_postgres/client.py index a6ae564c..3f20e7b3 100644 --- a/tap_postgres/client.py +++ b/tap_postgres/client.py @@ -30,7 +30,6 @@ from collections.abc import Iterable, Mapping from singer_sdk.helpers.types import Context - from sqlalchemy.dialects import postgresql from sqlalchemy.engine import Engine from sqlalchemy.engine.reflection import Inspector @@ -38,13 +37,29 @@ class PostgresSQLToJSONSchema(SQLToJSONSchema): """Custom SQL to JSON Schema conversion for Postgres.""" - def __init__(self, dates_as_string: bool, json_as_object: bool, *args, **kwargs): + def __init__(self, *, dates_as_string: bool, json_as_object: bool, **kwargs): """Initialize the SQL to JSON Schema converter.""" - super().__init__(*args, **kwargs) + super().__init__(**kwargs) self.dates_as_string = dates_as_string self.json_as_object = json_as_object - @SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined] + @classmethod + def from_config(cls, config: dict) -> PostgresSQLToJSONSchema: + """Instantiate the SQL to JSON Schema converter from a config dictionary.""" + return cls( + dates_as_string=config["dates_as_string"], + json_as_object=config["json_as_object"], + ) + + @functools.singledispatchmethod + def to_jsonschema(self, column_type: Any) -> dict: + """Override the default mapping for NUMERIC columns. + + For example, a scale of 4 translates to a multipleOf 0.0001. + """ + return super().to_jsonschema(column_type) + + @to_jsonschema.register def array_to_jsonschema(self, column_type: postgresql.ARRAY) -> dict: """Override the default mapping for NUMERIC columns. @@ -55,21 +70,21 @@ def array_to_jsonschema(self, column_type: postgresql.ARRAY) -> dict: "items": self.to_jsonschema(column_type.item_type), } - @SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined] + @to_jsonschema.register def json_to_jsonschema(self, column_type: postgresql.JSON) -> dict: """Override the default mapping for JSON and JSONB columns.""" if self.json_as_object: return {"type": ["object", "null"]} return {"type": ["string", "number", "integer", "array", "object", "boolean"]} - @SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined] + @to_jsonschema.register def datetime_to_jsonschema(self, column_type: sqlalchemy.types.DateTime) -> dict: """Override the default mapping for DATETIME columns.""" if self.dates_as_string: return {"type": ["string", "null"]} return super().datetime_to_jsonschema(column_type) - @SQLToJSONSchema.to_jsonschema.register # type: ignore[attr-defined] + @to_jsonschema.register def date_to_jsonschema(self, column_type: sqlalchemy.types.Date) -> dict: """Override the default mapping for DATE columns.""" if self.dates_as_string: @@ -133,6 +148,8 @@ def patched_conform( class PostgresConnector(SQLConnector): """Connects to the Postgres SQL source.""" + sql_to_jsonschema_converter = PostgresSQLToJSONSchema + def __init__( self, config: dict | None = None, @@ -161,14 +178,6 @@ def __init__( super().__init__(config=config, sqlalchemy_url=sqlalchemy_url) - @functools.cached_property - def sql_to_jsonschema(self): - """Return a mapping of SQL types to JSON Schema types.""" - return PostgresSQLToJSONSchema( - dates_as_string=self.config["dates_as_string"], - json_as_object=self.config["json_as_object"], - ) - def get_schema_names(self, engine: Engine, inspected: Inspector) -> list[str]: """Return a list of schema names in DB, or overrides with user-provided values.