Merge pull request #94 from ydb-platform/extended_dates_format

Date32, Datetime64 and Timestamp64 support

Author: vgvoleg

docs/types.rst

@@ -12,97 +12,134 @@ For more information about YDB data types, see the `YDB Type System Documentatio
 Type Mapping Summary
 --------------------
 
-The following table shows the complete mapping between YDB native types, SQLAlchemy types, and Python types:
+The following table shows the complete mapping between YDB native types, YDB SQLAlchemy types, standard SQLAlchemy types, and Python types:
 
 .. list-table:: YDB Type System Reference
    :header-rows: 1
-   :widths: 20 25 20 35
+   :widths: 15 20 20 15 30
 
    * - YDB Native Type
-     - SQLAlchemy Type
+     - YDB SA Type
+     - SA Type
      - Python Type
      - Notes
    * - ``Bool``
-     - ``BOOLEAN``
+     -
+     - ``Boolean``
      - ``bool``
      -
    * - ``Int8``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.Int8`
      -
      - ``int``
      - -2^7 to 2^7-1
    * - ``Int16``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.Int16`
      -
      - ``int``
      - -2^15 to 2^15-1
    * - ``Int32``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.Int32`
      -
      - ``int``
      - -2^31 to 2^31-1
    * - ``Int64``
-     - ``INTEGER``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.Int64`
+     - ``Integer``
      - ``int``
-     - -2^63 to 2^63-1
+     - -2^63 to 2^63-1, default integer type
    * - ``Uint8``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.UInt8`
      -
      - ``int``
      - 0 to 2^8-1
    * - ``Uint16``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.UInt16`
      -
      - ``int``
      - 0 to 2^16-1
    * - ``Uint32``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.UInt32`
      -
      - ``int``
      - 0 to 2^32-1
    * - ``Uint64``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.UInt64`
      -
      - ``int``
      - 0 to 2^64-1
    * - ``Float``
-     - ``FLOAT``
+     -
+     - ``Float``
      - ``float``
      -
    * - ``Double``
+     -
      - ``Double``
      - ``float``
      - Available in SQLAlchemy 2.0+
    * - ``Decimal(p,s)``
-     - ``DECIMAL`` / ``NUMERIC``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.Decimal`
+     - ``DECIMAL``
      - ``decimal.Decimal``
      -
    * - ``String``
-     - ``BINARY`` / ``BLOB``
-     - ``str`` / ``bytes``
+     -
+     - ``BINARY``
+     - ``bytes``
      -
    * - ``Utf8``
-     - ``CHAR`` / ``VARCHAR`` / ``TEXT`` / ``NVARCHAR``
+     -
+     - ``String`` / ``Text``
      - ``str``
      -
    * - ``Date``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDate`
      - ``Date``
      - ``datetime.date``
      -
+   * - ``Date32``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDate32`
+     -
+     - ``datetime.date``
+     - Extended date range support
    * - ``Datetime``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDateTime`
      - ``DATETIME``
      - ``datetime.datetime``
      -
+   * - ``Datetime64``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDateTime64`
+     -
+     - ``datetime.datetime``
+     - Extended datetime range
    * - ``Timestamp``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlTimestamp`
      - ``TIMESTAMP``
      - ``datetime.datetime``
      -
+   * - ``Timestamp64``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlTimestamp64`
+     -
+     - ``datetime.datetime``
+     - Extended timestamp range
    * - ``Json``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.json.YqlJSON`
      - ``JSON``
      - ``dict`` / ``list``
      -
    * - ``List<T>``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.ListType`
      - ``ARRAY``
      - ``list``
      -
    * - ``Struct<...>``
+     - :class:`~ydb_sqlalchemy.sqlalchemy.types.StructType`
      -
      - ``dict``
      -
    * - ``Optional<T>``
+     -
      - ``nullable=True``
      - ``None`` + base type
      -
@@ -145,6 +182,10 @@ YDB provides specific integer types with defined bit widths:
        byte_value = Column(UInt8)             # Unsigned 8-bit integer (0-255)
        counter = Column(UInt32)               # Unsigned 32-bit integer
 
+For detailed API reference, see:
+:class:`~ydb_sqlalchemy.sqlalchemy.types.Int8`, :class:`~ydb_sqlalchemy.sqlalchemy.types.Int16`, :class:`~ydb_sqlalchemy.sqlalchemy.types.Int32`, :class:`~ydb_sqlalchemy.sqlalchemy.types.Int64`,
+:class:`~ydb_sqlalchemy.sqlalchemy.types.UInt8`, :class:`~ydb_sqlalchemy.sqlalchemy.types.UInt16`, :class:`~ydb_sqlalchemy.sqlalchemy.types.UInt32`, :class:`~ydb_sqlalchemy.sqlalchemy.types.UInt64`.
+
 Decimal Type
 ------------
 
@@ -176,14 +217,19 @@ YDB supports high-precision decimal numbers:
        percentage=99.99
    ))
 
+For detailed API reference, see: :class:`~ydb_sqlalchemy.sqlalchemy.types.Decimal`.
+
 Date and Time Types
 -------------------
 
 YDB provides several date and time types:
 
 .. code-block:: python
 
-   from ydb_sqlalchemy.sqlalchemy.types import YqlDate, YqlDateTime, YqlTimestamp
+   from ydb_sqlalchemy.sqlalchemy.types import (
+       YqlDate, YqlDateTime, YqlTimestamp,
+       YqlDate32, YqlDateTime64, YqlTimestamp64
+   )
    from sqlalchemy import DateTime
    import datetime
 
@@ -192,15 +238,24 @@ YDB provides several date and time types:
 
        id = Column(UInt64, primary_key=True)
 
-       # Date only (YYYY-MM-DD)
+       # Date only (YYYY-MM-DD) - standard range
        event_date = Column(YqlDate)
 
-       # DateTime with timezone support
+       # Date32 - extended date range support
+       extended_date = Column(YqlDate32)
+
+       # DateTime with timezone support - standard range
        created_at = Column(YqlDateTime(timezone=True))
 
-       # Timestamp (high precision)
+       # DateTime64 - extended range
+       precise_datetime = Column(YqlDateTime64(timezone=True))
+
+       # Timestamp (high precision) - standard range
        precise_time = Column(YqlTimestamp)
 
+       # Timestamp64 - extended range with microsecond precision
+       extended_timestamp = Column(YqlTimestamp64)
+
        # Standard SQLAlchemy DateTime also works
        updated_at = Column(DateTime)
 
@@ -211,7 +266,14 @@ YDB provides several date and time types:
    session.add(EventLog(
        id=1,
        event_date=today,
+       extended_date=today,
        created_at=now,
+       precise_datetime=now,
        precise_time=now,
+       extended_timestamp=now,
        updated_at=now
    ))
+
+For detailed API reference, see:
+:class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDate`, :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDateTime`, :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlTimestamp`,
+:class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDate32`, :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlDateTime64`, :class:`~ydb_sqlalchemy.sqlalchemy.datetime_types.YqlTimestamp64`.

requirements.txt

@@ -1,3 +1,3 @@
 sqlalchemy >= 1.4.0, < 3.0.0
-ydb >= 3.18.8
+ydb >= 3.21.6
 ydb-dbapi >= 0.1.10

test/test_suite.py

@@ -1,4 +1,5 @@
 import ctypes
+import datetime
 import decimal
 
 import pytest
@@ -424,6 +425,16 @@ class DateTest(_DateTest):
     run_dispose_bind = "once"
 
 
+class Date32Test(_DateTest):
+    run_dispose_bind = "once"
+    datatype = ydb_sa_types.YqlDate32
+    data = datetime.date(1969, 1, 1)
+
+    @pytest.mark.skip("Default binding for DATE is not compatible with Date32")
+    def test_select_direct(self, connection):
+        pass
+
+
 class DateTimeMicrosecondsTest(_DateTimeMicrosecondsTest):
     run_dispose_bind = "once"
 
@@ -432,10 +443,30 @@ class DateTimeTest(_DateTimeTest):
     run_dispose_bind = "once"
 
 
+class DateTime64Test(_DateTimeTest):
+    datatype = ydb_sa_types.YqlDateTime64
+    data = datetime.datetime(1969, 10, 15, 12, 57, 18)
+    run_dispose_bind = "once"
+
+    @pytest.mark.skip("Default binding for DATETIME is not compatible with DateTime64")
+    def test_select_direct(self, connection):
+        pass
+
+
 class TimestampMicrosecondsTest(_TimestampMicrosecondsTest):
     run_dispose_bind = "once"
 
 
+class Timestamp64MicrosecondsTest(_TimestampMicrosecondsTest):
+    run_dispose_bind = "once"
+    datatype = ydb_sa_types.YqlTimestamp64
+    data = datetime.datetime(1969, 10, 15, 12, 57, 18, 396)
+
+    @pytest.mark.skip("Default binding for TIMESTAMP is not compatible with Timestamp64")
+    def test_select_direct(self, connection):
+        pass
+
+
 @pytest.mark.skip("unsupported Time data type")
 class TimeTest(_TimeTest):
     pass

ydb_sqlalchemy/sqlalchemy/__init__.py

@@ -61,6 +61,9 @@ def upsert(table):
     ydb.DecimalType: sa.DECIMAL,
     ydb.PrimitiveType.Yson: sa.TEXT,
     ydb.PrimitiveType.Date: sa.DATE,
+    ydb.PrimitiveType.Date32: sa.DATE,
+    ydb.PrimitiveType.Timestamp64: sa.TIMESTAMP,
+    ydb.PrimitiveType.Datetime64: sa.DATETIME,
     ydb.PrimitiveType.Datetime: sa.DATETIME,
     ydb.PrimitiveType.Timestamp: sa.TIMESTAMP,
     ydb.PrimitiveType.Interval: sa.INTEGER,

ydb_sqlalchemy/sqlalchemy/compiler/base.py

@@ -135,6 +135,15 @@ def visit_DATETIME(self, type_: sa.DATETIME, **kw):
     def visit_TIMESTAMP(self, type_: sa.TIMESTAMP, **kw):
         return "Timestamp"
 
+    def visit_date32(self, type_: types.YqlDate32, **kw):
+        return "Date32"
+
+    def visit_timestamp64(self, type_: types.YqlTimestamp64, **kw):
+        return "Timestamp64"
+
+    def visit_datetime64(self, type_: types.YqlDateTime64, **kw):
+        return "DateTime64"
+
     def visit_list_type(self, type_: types.ListType, **kw):
         inner = self.process(type_.item_type, **kw)
         return f"List<{inner}>"
@@ -193,6 +202,12 @@ def get_ydb_type(
         elif isinstance(type_, types.YqlJSON.YqlJSONPathType):
             ydb_type = ydb.PrimitiveType.Utf8
         # Json
+        elif isinstance(type_, types.YqlDate32):
+            ydb_type = ydb.PrimitiveType.Date32
+        elif isinstance(type_, types.YqlTimestamp64):
+            ydb_type = ydb.PrimitiveType.Timestamp64
+        elif isinstance(type_, types.YqlDateTime64):
+            ydb_type = ydb.PrimitiveType.Datetime64
         elif isinstance(type_, sa.DATETIME):
             ydb_type = ydb.PrimitiveType.Datetime
         elif isinstance(type_, sa.TIMESTAMP):

ydb_sqlalchemy/sqlalchemy/datetime_types.py

@@ -36,3 +36,39 @@ def process(value: Optional[datetime.datetime]) -> Optional[int]:
             return int(value.timestamp())
 
         return process
+
+
+class YqlDate32(YqlDate):
+    __visit_name__ = "date32"
+
+    def literal_processor(self, dialect):
+        parent = super().literal_processor(dialect)
+
+        def process(value):
+            return f"Date32({parent(value)})"
+
+        return process
+
+
+class YqlTimestamp64(YqlTimestamp):
+    __visit_name__ = "timestamp64"
+
+    def literal_processor(self, dialect):
+        parent = super().literal_processor(dialect)
+
+        def process(value):
+            return f"Timestamp64({parent(value)})"
+
+        return process
+
+
+class YqlDateTime64(YqlDateTime):
+    __visit_name__ = "datetime64"
+
+    def literal_processor(self, dialect):
+        parent = super().literal_processor(dialect)
+
+        def process(value):
+            return f"DateTime64({parent(value)})"
+
+        return process

ydb_sqlalchemy/sqlalchemy/types.py

@@ -11,7 +11,7 @@
 from sqlalchemy import ARRAY, exc, types
 from sqlalchemy.sql import type_api
 
-from .datetime_types import YqlDate, YqlDateTime, YqlTimestamp  # noqa: F401
+from .datetime_types import YqlDate, YqlDateTime, YqlTimestamp, YqlDate32, YqlTimestamp64, YqlDateTime64  # noqa: F401
 from .json import YqlJSON  # noqa: F401