Enable SDC metadata in SQLite tests

.pre-commit-config.yaml

@@ -44,7 +44,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.0
+  rev: v0.8.1
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

samples/sample_tap_gitlab/gitlab_rest_streams.py

@@ -74,7 +74,7 @@ def partitions(self) -> list[dict]:
         if "{project_id}" in self.path:
             return [
                 {"project_id": pid}
-                for pid in t.cast(list, self.config.get("project_ids"))
+                for pid in t.cast("list", self.config.get("project_ids"))
             ]
         if "{group_id}" in self.path:
             if "group_ids" not in self.config:
@@ -84,7 +84,8 @@ def partitions(self) -> list[dict]:
                 )
                 raise ValueError(msg)
             return [
-                {"group_id": gid} for gid in t.cast(list, self.config.get("group_ids"))
+                {"group_id": gid}
+                for gid in t.cast("list", self.config.get("group_ids"))
             ]
         msg = (
             f"Could not detect partition type for Gitlab stream '{self.name}' "

samples/sample_target_sqlite/__init__.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import datetime
+import sqlite3
 import typing as t
 
 from singer_sdk import SQLConnector, SQLSink, SQLTarget
@@ -10,6 +12,46 @@
 DB_PATH_CONFIG = "path_to_db"
 
 
+def adapt_date_iso(val):
+    """Adapt datetime.date to ISO 8601 date."""
+    return val.isoformat()
+
+
+def adapt_datetime_iso(val):
+    """Adapt datetime.datetime to timezone-naive ISO 8601 date."""
+    return val.isoformat()
+
+
+def adapt_datetime_epoch(val):
+    """Adapt datetime.datetime to Unix timestamp."""
+    return int(val.timestamp())
+
+
+sqlite3.register_adapter(datetime.date, adapt_date_iso)
+sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso)
+sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch)
+
+
+def convert_date(val):
+    """Convert ISO 8601 date to datetime.date object."""
+    return datetime.date.fromisoformat(val.decode())
+
+
+def convert_datetime(val):
+    """Convert ISO 8601 datetime to datetime.datetime object."""
+    return datetime.datetime.fromisoformat(val.decode())
+
+
+def convert_timestamp(val):
+    """Convert Unix epoch timestamp to datetime.datetime object."""
+    return datetime.datetime.fromtimestamp(int(val), tz=datetime.timezone.utc)
+
+
+sqlite3.register_converter("date", convert_date)
+sqlite3.register_converter("datetime", convert_datetime)
+sqlite3.register_converter("timestamp", convert_timestamp)
+
+
 class SQLiteConnector(SQLConnector):
     """The connector for SQLite.
 

tests/samples/conftest.py

@@ -15,7 +15,7 @@
 @pytest.fixture
 def csv_config(outdir: str) -> dict:
     """Get configuration dictionary for target-csv."""
-    return {"target_folder": outdir}
+    return {"target_folder": outdir, "add_record_metadata": False}
 
 
 @pytest.fixture

tests/samples/test_tap_sqlite.py

@@ -10,7 +10,6 @@
 
 from samples.sample_tap_sqlite import SQLiteTap
 from samples.sample_target_csv.csv_target import SampleTargetCSV
-from singer_sdk import SQLStream
 from singer_sdk._singerlib import MetadataMapping, StreamMetadata
 from singer_sdk.testing import (
     get_standard_tap_tests,
@@ -21,6 +20,7 @@
 if t.TYPE_CHECKING:
     from pathlib import Path
 
+    from singer_sdk import SQLStream
     from singer_sdk.tap_base import SQLTap
 
 
@@ -50,7 +50,7 @@ def test_tap_sqlite_cli(sqlite_sample_db_config: dict[str, t.Any], tmp_path: Pat
 
 
 def test_sql_metadata(sqlite_sample_tap: SQLTap):
-    stream = t.cast(SQLStream, sqlite_sample_tap.streams["main-t1"])
+    stream = t.cast("SQLStream", sqlite_sample_tap.streams["main-t1"])
     detected_metadata = stream.catalog_entry["metadata"]
     detected_root_md = next(md for md in detected_metadata if md["breadcrumb"] == [])
     detected_root_md = detected_root_md["metadata"]
@@ -68,7 +68,7 @@ def test_sql_metadata(sqlite_sample_tap: SQLTap):
 def test_sqlite_discovery(sqlite_sample_tap: SQLTap):
     _discover_and_select_all(sqlite_sample_tap)
     sqlite_sample_tap.sync_all()
-    stream = t.cast(SQLStream, sqlite_sample_tap.streams["main-t1"])
+    stream = t.cast("SQLStream", sqlite_sample_tap.streams["main-t1"])
     schema = stream.schema
     assert len(schema["properties"]) == 3
     assert stream.name == stream.tap_stream_id == "main-t1"
@@ -89,7 +89,7 @@ def test_sqlite_discovery(sqlite_sample_tap: SQLTap):
 
 def test_sqlite_input_catalog(sqlite_sample_tap: SQLTap):
     sqlite_sample_tap.sync_all()
-    stream = t.cast(SQLStream, sqlite_sample_tap.streams["main-t1"])
+    stream = t.cast("SQLStream", sqlite_sample_tap.streams["main-t1"])
     assert len(stream.schema["properties"]) == 3
     assert len(stream.stream_maps[0].transformed_schema["properties"]) == 3
 

tests/samples/test_target_sqlite.py

@@ -38,7 +38,7 @@ def path_to_target_db(tmp_path: Path) -> Path:
 @pytest.fixture
 def sqlite_target_test_config(path_to_target_db: Path) -> dict:
     """Get configuration dictionary for target-csv."""
-    return {"path_to_db": str(path_to_target_db)}
+    return {"path_to_db": str(path_to_target_db), "add_record_metadata": True}
 
 
 @pytest.fixture
@@ -505,8 +505,8 @@ def test_record_with_missing_properties(
             dedent(
                 """\
                 INSERT INTO test_stream
-                (id, name, "table")
-                VALUES (:id, :name, :table)""",
+                (id, name, "table", _sdc_extracted_at, _sdc_received_at, _sdc_batched_at, _sdc_deleted_at, _sdc_sequence, _sdc_table_version, _sdc_sync_started_at)
+                VALUES (:id, :name, :table, :_sdc_extracted_at, :_sdc_received_at, :_sdc_batched_at, :_sdc_deleted_at, :_sdc_sequence, :_sdc_table_version, :_sdc_sync_started_at)""",  # noqa: E501
             ),
         ),
     ],
@@ -563,6 +563,13 @@ def test_hostile_to_sqlite(
     )
     columns = {res[0] for res in cursor.fetchall()}
     assert columns == {
+        "_sdc_batched_at",
+        "_sdc_deleted_at",
+        "_sdc_extracted_at",
+        "_sdc_received_at",
+        "_sdc_sequence",
+        "_sdc_sync_started_at",
+        "_sdc_table_version",
         "name_with_spaces",
         "nameiscamelcase",
         "name_with_dashes",