Merge branch 'main' into edgarrmondragon/refactor/capabilities

Author: edgarrmondragon

.github/ISSUE_TEMPLATE/bug.yml

@@ -67,18 +67,10 @@ body:
     id: what-happened
     attributes:
       label: Description
-      description: Describe what you were trying to get done
+      description: Describe what you were trying to get done, what you expected to happen, and what actually happened. Include any error messages you received, or a snippet of the code that caused the issue.
       placeholder: Tell us what happened, what went wrong, and what you expected to happen
     validations:
       required: true
-  - type: textarea
-    id: failing-code
-    attributes:
-      label: Code
-      description: Paste the failing code and/or traceback, if applicable
-      render: Python
-    validations:
-      required: false
   - type: input
     id: slack_or_linen
     attributes:

.pre-commit-config.yaml

@@ -37,14 +37,14 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.4
+  rev: 0.30.0
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.0
+  rev: v0.8.1
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import decimal
 import typing as t
 
 import requests  # noqa: TCH002
@@ -61,7 +62,7 @@ def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
             Each record from the source.
         """
         # TODO: Parse response body and return a set of records.
-        resp_json = response.json()
+        resp_json = response.json(parse_float=decimal.Decimal)
         yield from resp_json.get("<TODO>")
 
     def post_process(

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import decimal
 import typing as t
 {% if cookiecutter.auth_method in ("OAuth2", "JWT") -%}
 from functools import cached_property
@@ -204,7 +205,10 @@ def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
             Each record from the source.
         """
         # TODO: Parse response body and return a set of records.
-        yield from extract_jsonpath(self.records_jsonpath, input=response.json())
+        yield from extract_jsonpath(
+            self.records_jsonpath,
+            input=response.json(parse_float=decimal.Decimal),
+        )
 
     def post_process(
         self,

docs/guides/sql-tap.md

@@ -24,15 +24,19 @@ from my_sqlalchemy_dialect import VectorType
 
 
 class CustomSQLToJSONSchema(SQLToJSONSchema):
-    @SQLToJSONSchema.to_jsonschema.register
+    @functools.singledispatchmethod
+    def to_jsonschema(self, column_type):
+        return super().to_jsonschema(column_type)
+
+    @to_jsonschema.register
     def custom_number_to_jsonschema(self, column_type: Numeric):
         """Override the default mapping for NUMERIC columns.
 
         For example, a scale of 4 translates to a multipleOf 0.0001.
         """
         return {"type": ["number"], "multipleOf": 10**-column_type.scale}
 
-    @SQLToJSONSchema.to_jsonschema.register(VectorType)
+    @to_jsonschema.register(VectorType)
     def vector_to_json_schema(self, column_type):
         """Custom vector to JSON schema."""
         return th.ArrayType(th.NumberType()).to_dict()
@@ -42,7 +46,7 @@ class CustomSQLToJSONSchema(SQLToJSONSchema):
 You can also use a type annotation to specify the type of the column when registering a new method:
 
 ```python
-@SQLToJSONSchema.to_jsonschema.register
+@to_jsonschema.register
 def vector_to_json_schema(self, column_type: VectorType):
     return th.ArrayType(th.NumberType()).to_dict()
 ```
@@ -52,7 +56,23 @@ Then, you need to use your custom type mapping in your connector:
 
 ```python
 class MyConnector(SQLConnector):
-    @functools.cached_property
-    def sql_to_jsonschema(self):
-        return CustomSQLToJSONSchema()
+    sql_to_jsonschema_converter = CustomSQLToJSONSchema
+```
+
+### Adapting the type mapping based on user configuration
+
+
+If your type mapping depends on some user-defined configuration, you can also override the `from_config` method to pass the configuration to your custom type mapping:
+
+```python
+class ConfiguredSQLToJSONSchema(SQLToJSONSchema):
+    def __init__(self, *, my_custom_setting: str, **kwargs):
+        super().__init__(**kwargs)
+        self.my_custom_setting = my_custom_setting
+
+    @classmethod
+    def from_config(cls, config: dict):
+        return cls(my_custom_setting=config.get("my_custom_setting", "default_value"))
 ```
+
+Then, you can use your custom type mapping in your connector as in the previous example.

samples/sample_tap_gitlab/gitlab_rest_streams.py

@@ -74,7 +74,7 @@ def partitions(self) -> list[dict]:
         if "{project_id}" in self.path:
             return [
                 {"project_id": pid}
-                for pid in t.cast(list, self.config.get("project_ids"))
+                for pid in t.cast("list", self.config.get("project_ids"))
             ]
         if "{group_id}" in self.path:
             if "group_ids" not in self.config:
@@ -84,7 +84,8 @@ def partitions(self) -> list[dict]:
                 )
                 raise ValueError(msg)
             return [
-                {"group_id": gid} for gid in t.cast(list, self.config.get("group_ids"))
+                {"group_id": gid}
+                for gid in t.cast("list", self.config.get("group_ids"))
             ]
         msg = (
             f"Could not detect partition type for Gitlab stream '{self.name}' "

samples/sample_target_sqlite/__init__.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import datetime
+import sqlite3
 import typing as t
 
 from singer_sdk import SQLConnector, SQLSink, SQLTarget
@@ -10,6 +12,46 @@
 DB_PATH_CONFIG = "path_to_db"
 
 
+def adapt_date_iso(val):
+    """Adapt datetime.date to ISO 8601 date."""
+    return val.isoformat()
+
+
+def adapt_datetime_iso(val):
+    """Adapt datetime.datetime to timezone-naive ISO 8601 date."""
+    return val.isoformat()
+
+
+def adapt_datetime_epoch(val):
+    """Adapt datetime.datetime to Unix timestamp."""
+    return int(val.timestamp())
+
+
+sqlite3.register_adapter(datetime.date, adapt_date_iso)
+sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso)
+sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch)
+
+
+def convert_date(val):
+    """Convert ISO 8601 date to datetime.date object."""
+    return datetime.date.fromisoformat(val.decode())
+
+
+def convert_datetime(val):
+    """Convert ISO 8601 datetime to datetime.datetime object."""
+    return datetime.datetime.fromisoformat(val.decode())
+
+
+def convert_timestamp(val):
+    """Convert Unix epoch timestamp to datetime.datetime object."""
+    return datetime.datetime.fromtimestamp(int(val), tz=datetime.timezone.utc)
+
+
+sqlite3.register_converter("date", convert_date)
+sqlite3.register_converter("datetime", convert_datetime)
+sqlite3.register_converter("timestamp", convert_timestamp)
+
+
 class SQLiteConnector(SQLConnector):
     """The connector for SQLite.
 

singer_sdk/connectors/sql.py

@@ -121,8 +121,37 @@ class SQLToJSONSchema:
     This class provides a mapping from SQLAlchemy types to JSON Schema types.
 
     .. versionadded:: 0.41.0
+    .. versionchanged:: 0.43.0
+       Added the :meth:`singer_sdk.connectors.sql.SQLToJSONSchema.from_config` class
+       method.
     """
 
+    @classmethod
+    def from_config(cls: type[SQLToJSONSchema], config: dict) -> SQLToJSONSchema:  # noqa: ARG003
+        """Create a new instance from a configuration dictionary.
+
+        Override this to instantiate this converter with values from the tap's
+        configuration dictionary.
+
+        .. code-block:: python
+
+           class CustomSQLToJSONSchema(SQLToJSONSchema):
+               def __init__(self, *, my_custom_option, **kwargs):
+                   super().__init__(**kwargs)
+                   self.my_custom_option = my_custom_option
+
+               @classmethod
+               def from_config(cls, config):
+                   return cls(my_custom_option=config.get("my_custom_option"))
+
+        Args:
+            config: The configuration dictionary.
+
+        Returns:
+            A new instance of the class.
+        """
+        return cls()
+
     @functools.singledispatchmethod
     def to_jsonschema(self, column_type: sa.types.TypeEngine) -> dict:  # noqa: ARG002, D102, PLR6301
         return th.StringType.type_dict  # type: ignore[no-any-return]
@@ -453,6 +482,11 @@ class SQLConnector:  # noqa: PLR0904
     #: The absolute maximum length for VARCHAR columns that the database supports.
     max_varchar_length: int | None = None
 
+    #: The SQL-to-JSON type mapper class for this SQL connector. Override this property
+    #: with a subclass of :class:`~singer_sdk.connectors.sql.SQLToJSONSchema` to provide
+    #: a custom mapping for your SQL dialect.
+    sql_to_jsonschema_converter: type[SQLToJSONSchema] = SQLToJSONSchema
+
     def __init__(
         self,
         config: dict | None = None,
@@ -493,7 +527,7 @@ def sql_to_jsonschema(self) -> SQLToJSONSchema:
 
         .. versionadded:: 0.41.0
         """
-        return SQLToJSONSchema()
+        return self.sql_to_jsonschema_converter.from_config(self.config)
 
     @functools.cached_property
     def jsonschema_to_sql(self) -> JSONSchemaToSQL:
@@ -607,7 +641,7 @@ def get_sqlalchemy_url(self, config: dict[str, t.Any]) -> str:  # noqa: PLR6301
             msg = "Could not find or create 'sqlalchemy_url' for connection."
             raise ConfigValidationError(msg)
 
-        return t.cast(str, config["sqlalchemy_url"])
+        return t.cast("str", config["sqlalchemy_url"])
 
     def to_jsonschema_type(
         self,
@@ -1395,7 +1429,7 @@ def _get_type_sort_key(
 
             _len = int(getattr(sql_type, "length", 0) or 0)
 
-            _pytype = t.cast(type, sql_type.python_type)
+            _pytype = t.cast("type", sql_type.python_type)
             if issubclass(_pytype, (str, bytes)):
                 return 900, _len
             if issubclass(_pytype, datetime):