fix: Map real/float4 types to JSON schema number (#450)

https://www.postgresql.org/docs/current/datatype.html
MeltanoLabs · Jul 3, 2024 · ab1cbae · ab1cbae
1 parent b7d8b2b
commit ab1cbae
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 7 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -18,8 +18,14 @@ jobs:
     env:
       PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
 
     steps:
     - name: Checkout code

diff --git a/tap_postgres/client.py b/tap_postgres/client.py
@@ -212,6 +212,8 @@ def sdk_typing_object(
             "decimal": th.NumberType(),
             "double": th.NumberType(),
             "float": th.NumberType(),
+            "real": th.NumberType(),
+            "float4": th.NumberType(),
             "string": th.StringType(),
             "text": th.StringType(),
             "char": th.StringType(),

diff --git a/tests/test_core.py b/tests/test_core.py
@@ -339,7 +339,7 @@ def test_jsonb_array():
         assert test_runner.records[altered_table_name][i] == rows[i]
 
 
-def test_decimal():
+def test_numeric_types():
     """Schema was wrong for Decimal objects. Check they are correctly selected."""
     table_name = "test_decimal"
     engine = sa.create_engine(SAMPLE_CONFIG["sqlalchemy_url"], future=True)
@@ -348,20 +348,31 @@ def test_decimal():
     table = sa.Table(
         table_name,
         metadata_obj,
-        sa.Column("column", sa.Numeric()),
+        sa.Column("my_numeric", sa.Numeric()),
+        sa.Column("my_real", sa.REAL()),
     )
     with engine.begin() as conn:
         table.drop(conn, checkfirst=True)
         metadata_obj.create_all(conn)
-        insert = table.insert().values(column=decimal.Decimal("3.14"))
+        insert = table.insert().values(
+            my_numeric=decimal.Decimal("3.14"),
+            my_real=3.14,
+        )
         conn.execute(insert)
-        insert = table.insert().values(column=decimal.Decimal("12"))
+        insert = table.insert().values(
+            my_numeric=decimal.Decimal("12"),
+            my_real=12,
+        )
         conn.execute(insert)
-        insert = table.insert().values(column=decimal.Decimal("10000.00001"))
+        insert = table.insert().values(
+            my_numeric=decimal.Decimal("10000.00001"),
+            my_real=10000.00001,
+        )
         conn.execute(insert)
     tap = TapPostgres(config=SAMPLE_CONFIG)
     tap_catalog = json.loads(tap.catalog_json_text)
     altered_table_name = f"{DB_SCHEMA_NAME}-{table_name}"
+
     for stream in tap_catalog["streams"]:
         if stream.get("stream") and altered_table_name not in stream["stream"]:
             for metadata in stream["metadata"]:
@@ -376,12 +387,15 @@ def test_decimal():
         tap_class=TapPostgres, config=SAMPLE_CONFIG, catalog=tap_catalog
     )
     test_runner.sync_all()
+
     for schema_message in test_runner.schema_messages:
         if (
             "stream" in schema_message
             and schema_message["stream"] == altered_table_name
         ):
-            assert "number" in schema_message["schema"]["properties"]["column"]["type"]
+            props = schema_message["schema"]["properties"]
+            assert "number" in props["my_numeric"]["type"]
+            assert "number" in props["my_real"]["type"]
 
 
 def test_filter_schemas():