diff --git a/pynuodb/calendar.py b/pynuodb/calendar.py new file mode 100644 index 0000000..bae2311 --- /dev/null +++ b/pynuodb/calendar.py @@ -0,0 +1,132 @@ + +__doc__ = """ +From: https://aa.usno.navy.mil/downloads/c15_usb_online.pdf + +15.4.2 Rules for the Civil Use of the Gregorian Calendar + +The Gregorian calendar uses the same months with the numbers of days +as it predecessor, the Julian calendar (see Table 15.5). Days are +counted from the first day of each month. Years are counted from the +initial epoch defined by Dionysius Exiguus (see ยง 15.1.8), and each +begins on January 1. A common year has 365 days but a leap year has +366, with an intercalary day, designated February 29, preceding March +1. Leap years are determined according to the following rule: + + Every year that is exactly divisible by 4 is a leap year, except for + years that are exactly divisible by 100, but these centurial years + are leap years if they are exactly divisible by 400. + +As a result, the year 2000 was a leap year, whereas 1900 and 2100 are +not. + +The epoch of the Gregorian calendar, (1 January 1) was Monday, 1 +January 3 in the Julian calendar or Julian Day Number 1721426. + +The algorithm's for ymd2day and day2ymd are based off of. +- https://github.com/SETI/rms-julian/blob/main/julian/calendar.py +""" + +#EPOCH 1/1/1970 + +# from: https://aa.usno.navy.mil/data/JulianDate +# Julian Dates +# Sunday 1 B.C. February 29 00:00:00.0 1721116.500000 +# Monday 1 B.C. March 1 00:00:00.0 1721117.500000 +# Thursday A.D. 1970 January 1 00:00:00.0 2440587.500000 (day 0 for our calculations) +# Thursday A.D. 1582 October 4 00:00:00.0 2299159.500000 (last day of julian calendar) +# Friday A.D. 1582 October 15 00:00:00.0 2299160.500000 (first day of gregorian calendar) + +# used to calculate daynum (relative to 1970-01-01) for dates before and including 1582 Oct 4 +_FEB29_1BCE_JULIAN = 1721116 - 2440587 +# used to calculate daynum (relative to 1970-01-01) for dates after and including 1582 Oct 15 +_FEB29_1BCE_GREGORIAN = _FEB29_1BCE_JULIAN + 2 +# used to shift daynum to calculate (y,m,d) relative to 1/1/1 using julian calendar +_MAR01_1BCE_JULIAN = - (_FEB29_1BCE_JULIAN + 1) +# daynum when gregorian calendar went into effect relative to 1/1/1970 +_GREGORIAN_DAY1 = 2299160 - 2440587 + + +def ymd2day(year: int, month: int, day: int,validate: bool =False) -> int: + + """ + Converts given year , month, day to number of days since unix EPOCH. + year - between 0001-9999 + month - 1 - 12 + day - 1 - 31 (depending upon month and year) + The calculation will be based upon: + - Georgian Calendar for dates from and including 10/15/1582. + - Julian Calendar for dates before and including 10/4/1582. + Dates between the Julian Calendar and Georgian Calendar don't exist and + ValueError will be raised. + + If validate = true then ValueError is raised if year,month,day is not a valid + date and within year range. + """ + + mm = (month+9)%12 + yy = year - mm//10 + d = day + + day_as_int = year*10000+month*100+day + if day_as_int > 15821014: + # use Georgian Calendar, leap year every 4 years except centuries that are not divisible by 400 + # 1900 - not yeap year + # 2000 - yeap year + daynum = (365*yy + yy//4 - yy//100 + yy//400) + (mm * 306 + 5)//10 + d + _FEB29_1BCE_GREGORIAN + elif day_as_int < 15821005: + # Julian Calendar, leap year ever 4 years + daynum = (365*yy + yy//4) + (mm * 306 + 5)//10 + d + _FEB29_1BCE_JULIAN + else: + raise ValueError(f"Invalid date {year:04}-{month:02}-{day:02} not in Gregorian or Julian Calendar") + + if validate: + if day2ymd(daynum) != (year,month,day): + raise ValueError(f"Invalid date {year:04}-{month:02}-{day:02}") + return daynum + +def day2ymd(daynum: int) -> (int,int,int): + """ + Converts given day number relative to 1970-01-01 to a tuple (year,month,day). + + + The calculation will be based upon: + - Georgian Calendar for dates from and including 10/15/1582. + - Julian Calendar for dates before and including 10/4/1582. + + Dates between the Julian Calendar and Georgian Calendar do not exist. + + +----------------------------+ + | daynum | (year,month,day) | + |---------+------------------| + | 0 | (1970,1,1) | + | -141427 | (1582,10,15) | + | -141428 | (1582,10,4) | + | -719614 | (1,1,1) | + | 2932896 | (9999,12,31) | + +----------------------------+ + + """ + + # before 1/1/1 or after 9999/12/25 + # if daynum < -719164 or daynum > 2932896: + # raise ValueError(f"Invalid daynum {daynum} before 0001-01-01 or after 9999-12-31") + + # In Julian Calender 0001-01-03 is (JD 1721426). + if daynum < _GREGORIAN_DAY1: + g = daynum + _MAR01_1BCE_JULIAN + y = (100 * g + 75) // 36525 + doy = g - (365*y + y//4) + else: + # In Georgian Calender 0001-01-01 is (JD 1721426). + g = daynum + _MAR01_1BCE_JULIAN - 2 + y = (10000*g + 14780)//3652425 # 365.2425 avg. number days in year. + doy = g - (365*y + y//4 - y//100 + y//400) + if doy < 0: + y -= 1 + doy = g - (365*y + y//4 - y//100 + y//400) + m0 = (100 * doy + 52)//3060 + m = (m0+2)%12 + 1 + y += (m0+2)//12 + d = doy - (m0*306+5)//10 + 1 + return (y,m,d) + diff --git a/pynuodb/connection.py b/pynuodb/connection.py index 1765e1f..d8cff35 100644 --- a/pynuodb/connection.py +++ b/pynuodb/connection.py @@ -17,6 +17,7 @@ from os import getpid import time import xml.etree.ElementTree as ElementTree +import tzlocal try: from typing import Mapping, Optional, Tuple # pylint: disable=unused-import @@ -132,10 +133,22 @@ def __init__(self, database=None, # type: Optional[str] **kwargs) self.__session.doConnect(params) - params.update({'user': user, - 'timezone': time.strftime('%Z'), - 'clientProcessId': str(getpid())}) - + additional_params = {'user': user, 'clientProcessId': str(getpid()) } + timezone_name = None + for key in params: + if key.lower() == 'timezone': + timezone_name = params[key] + break + if not timezone_name: + # from doc: https://pypi.org/project/tzlocal/ + # You can also use tzlocal to get the name of your local + # timezone, but only if your system is configured to make + # that possible. tzlocal looks for the timezone name in + # /etc/timezone, /var/db/zoneinfo, /etc/sysconfig/clock + # and /etc/conf.d/clock. If your /etc/localtime is a + # symlink it can also extract the name from that symlink. + additional_params['timezone'] = tzlocal.get_localzone_name() + params.update(additional_params) self.__session.open_database(database, password, params) # Set auto commit to false by default per PEP 249 diff --git a/pynuodb/datatype.py b/pynuodb/datatype.py index bf88369..99a3409 100644 --- a/pynuodb/datatype.py +++ b/pynuodb/datatype.py @@ -37,14 +37,17 @@ except ImportError: pass -from datetime import datetime as Timestamp, date as Date, time as Time +from datetime import datetime as Timestamp, date as Date, time as Time, timezone as TimeZone from datetime import timedelta as TimeDelta import decimal import time +import tzlocal from .exception import DataError +from .calendar import ymd2day, day2ymd isP2 = sys.version[0] == '2' +localZoneInfo = tzlocal.get_localzone() class Binary(bytes): """A binary string. @@ -78,60 +81,134 @@ def string(self): """The old implementation of Binary provided this.""" return self - +TICKSDAY=86400 def DateFromTicks(ticks): # type: (int) -> Date """Convert ticks to a Date object.""" - return Date(*time.localtime(ticks)[:3]) + y,m,d = day2ymd(ticks//TICKSDAY) + return Date(year=y,month=m,day=d) -def TimeFromTicks(ticks, micro=0): +def TimeFromTicks(ticks, micro=0, zoneinfo=localZoneInfo ): # type: (int, int) -> Time """Convert ticks to a Time object.""" - return Time(*time.localtime(ticks)[3:6] + (micro,)) - -def TimestampFromTicks(ticks, micro=0): + if ticks >= TICKSDAY or ticks <= -TICKSDAY: + dt = TimestampFromTicks(ticks,micro,zoneinfo) + _time = dt.time() + else: + if ticks < 0 and micro: + ticks -= 1 + timeticks = ticks % TICKSDAY + hour = timeticks // 3600 + sec = timeticks % 3600 + min = (sec // 60 ) + sec %= 60 + micro %= 1000000 + + # convert time to standard time offset from utc for given timezone. Use + # today's date for offset calculation. + + today = Timestamp.now().date() + time = Time(hour=hour,minute=min,second=sec,microsecond=micro,tzinfo=TimeZone.utc) + tstamp = Timestamp.combine(today,time).astimezone(zoneinfo) + dst_offset = tstamp.dst() + if dst_offset: + tstamp -= dst_offset + _time = tstamp.time() + return _time + + + +def TimestampFromTicks(ticks, micro=0,zoneinfo=localZoneInfo): # type: (int, int) -> Timestamp """Convert ticks to a Timestamp object.""" - return Timestamp(*time.localtime(ticks)[:6] + (micro,)) + + day = ticks//TICKSDAY + y,m,d = day2ymd(day) + + timeticks = ticks % TICKSDAY + hour = timeticks // 3600 + sec = timeticks % 3600 + min = sec // 60 + sec %= 60 + + # this requires both utc and current session to be between year 1 and year 9999 inclusive. + # nuodb could store a timestamp that is east of utc where utc would be year 10000, for now + # let's just return NULL for that case. + if y < 10000: + dt = Timestamp(year=y,month=m,day=d,hour=hour,minute=min,second=sec,microsecond=micro,tzinfo=TimeZone.utc) + dt = dt.astimezone(zoneinfo) + else: + # shift one day. + dt = Timestamp(year=9999,month=12,day=31,hour=hour, + minute=min,second=sec,microsecond=micro,tzinfo=TimeZone.utc) + dt = dt.astimezone(zoneinfo) + # add day back. + dt += TimeDelta(days=1) + return dt def DateToTicks(value): # type: (Date) -> int """Convert a Date object to ticks.""" - timeStruct = Date(value.year, value.month, value.day).timetuple() - try: - return int(time.mktime(timeStruct)) - except Exception: - raise DataError("Year out of range") - - -def TimeToTicks(value): + day = ymd2day(value.year, value.month, value.day) + return day * TICKSDAY + +def packtime(seconds: int, microseconds: int) -> (int, int): + if microseconds: + ndiv=0 + msecs = microseconds + shiftr = 1000000 + shiftl = 1 + while (microseconds % shiftr): + shiftr //= 10 + shiftl *= 10 + ndiv +=1 + return ( seconds*shiftl + microseconds//shiftr, ndiv ) + else: + return (seconds, 0) + +def TimeToTicks(value,zoneinfo = localZoneInfo): # type: (Time) -> Tuple[int, int] """Convert a Time object to ticks.""" - timeStruct = TimeDelta(hours=value.hour, minutes=value.minute, - seconds=value.second, - microseconds=value.microsecond) + + # convert time to time relative to connection timezone + # using today as date. + + tstamp = Timestamp.combine(Date.today(),value) + if tstamp.tzinfo is None: + tstamp = tstamp.replace(tzinfo=zoneinfo) + else: + tstamp = tstamp.astimezone(zoneinfo) + + dst_offset = zoneinfo.dst(tstamp) + utc_offset = zoneinfo.utcoffset(tstamp) + std_offset = dst_offset - utc_offset + + timeStruct = TimeDelta(hours=tstamp.hour, minutes=tstamp.minute, + seconds=tstamp.second, + microseconds=tstamp.microsecond) timeDec = decimal.Decimal(str(timeStruct.total_seconds())) - return (int((timeDec + time.timezone) * 10**abs(timeDec.as_tuple()[2])), + return (int((timeDec + std_offset) * 10**abs(timeDec.as_tuple()[2])), abs(timeDec.as_tuple()[2])) -def TimestampToTicks(value): +def TimestampToTicks(value,zoneinfo = localZoneInfo): # type: (Timestamp) -> Tuple[int, int] """Convert a Timestamp object to ticks.""" - timeStruct = Timestamp(value.year, value.month, value.day, value.hour, - value.minute, value.second).timetuple() - try: - if not value.microsecond: - return (int(time.mktime(timeStruct)), 0) - micro = decimal.Decimal(value.microsecond) / decimal.Decimal(1000000) - t1 = decimal.Decimal(int(time.mktime(timeStruct))) + micro - tlen = len(str(micro)) - 2 - return (int(t1 * decimal.Decimal(int(10**tlen))), tlen) - except Exception: - raise DataError("Year out of range") + + # if naive timezone then leave date/time but change tzinfo to + # be connection's timezone. + if value.tzinfo is None: + value = value.replace(tzinfo=zoneinfo) + dt = value.astimezone(TimeZone.utc) + timesecs = ymd2day(dt.year,dt.month,dt.day) * TICKSDAY + timesecs += dt.hour * 3600 + timesecs += dt.minute * 60 + timesecs += dt.second + packedtime = packtime(timesecs,dt.microsecond) + return packedtime class TypeObject(object): @@ -191,3 +268,4 @@ def TypeObjectFromNuodb(nuodb_type_name): if obj is None: raise DataError('received unknown column type "%s"' % (name)) return obj + diff --git a/pynuodb/encodedsession.py b/pynuodb/encodedsession.py index c1e457d..b74a50d 100644 --- a/pynuodb/encodedsession.py +++ b/pynuodb/encodedsession.py @@ -15,6 +15,8 @@ import struct import decimal import sys +from zoneinfo import ZoneInfo +import tzlocal try: from typing import Any, Collection, Dict, Iterable, List # pylint: disable=unused-import @@ -37,10 +39,8 @@ from .crypt import BaseCipher, ClientPassword # pylint: disable=unused-import -isP2 = sys.version[0] == '2' REMOVE_FORMAT = 0 - class EncodedSession(Session): # pylint: disable=too-many-public-methods """Class for representing an encoded session with the database. @@ -118,6 +118,22 @@ def __init__(self, host, service='SQL2', options=None, **kwargs): self.__encryption = False super(EncodedSession, self).__init__(host, service=service, options=options, **kwargs) + self.__timezone_name = None + self.__timezone_info = None + + @property + def timezone_name(self): + return self.__timezone_name + + @property + def timezone_info(self): + return ZoneInfo(self.__timezone_name) + + @timezone_name.setter + def timezone_name(self,tzname): + self.__timezone_name = tzname + + def open_database(self, db_name, password, parameters): # type: (str, str, Dict[str, str]) -> None @@ -128,6 +144,7 @@ def open_database(self, db_name, password, parameters): :param password: The user's password. :param parameters: Connection parameters. """ + params = parameters.copy() if 'clientInfo' not in params: params['clientInfo'] = 'pynuodb' @@ -173,6 +190,10 @@ def open_database(self, db_name, password, parameters): if cp: self._srp_handshake(params['user'], password, serverKey, salt, cp) + self._set_timezone() + + + def _srp_handshake(self, username, password, serverKey, salt, cp): # type: (str, str, str, str, ClientPassword) -> None """Authenticate the SRP session.""" @@ -251,6 +272,31 @@ def set_encryption(self, value): """Enable or disable encryption.""" self.__encryption = value + def _set_timezone(self): + """ + Query TE for TimeZone name. This is done because timezone abbreviations + are allowed in TE but, not handled by ZoneInfo. If TE gets a TimeZone=EST + connection property, it will set TimeZone system connection property to America/ + + # type: () -> None + """ + # Create a statement handle + self._putMessageId(protocol.CREATE) + self._exchangeMessages() + handle = self.getInt() + + self._setup_statement(handle, protocol.EXECUTEQUERY) + self.putString("select value from system.connectionproperties where property='TimeZone'") + self._exchangeMessages() + + # returns: rsHandle, count, colname, result, fieldValue, r2 + res = [self.getInt(), self.getInt(), self.getString(), + self.getInt(), self.getString(), self.getInt()] + self.timezone_name=res[-2] + self._putMessageId(protocol.CLOSESTATEMENT).putInt(handle) + + + def test_connection(self): # type: () -> None """Test this connection to the database.""" @@ -293,6 +339,11 @@ def execute_statement(self, statement, query): result = self.getInt() rowcount = self.getInt() + if self.__sessionVersion >= protocol.TIMESTAMP_WITHOUT_TZ: + tzChange = self.getBoolean() + if tzChange: + tzName = self.getString() + self.timezone_name = tzName return ExecutionResult(statement, result, rowcount) def close_statement(self, statement): @@ -338,6 +389,12 @@ def execute_prepared_statement(self, prepared_statement, parameters): result = self.getInt() rowcount = self.getInt() + if self.__sessionVersion >= protocol.TIMESTAMP_WITHOUT_TZ: + tzChange = self.getBoolean() + if tzChange: + tzName = self.getString() + self.timezone_name = tzName + return ExecutionResult(prepared_statement, result, rowcount) def execute_batch_prepared_statement(self, prepared_statement, param_lists): @@ -375,6 +432,17 @@ def execute_batch_prepared_statement(self, prepared_statement, param_lists): if error_string is not None: raise BatchError(error_string, results) + if self.__sessionVersion >= protocol.TIMESTAMP_WITHOUT_TZ: + tzChange = self.getBoolean() + if tzChange: + tzName = self.getString() + self.timezone_name = tzName + + # timezone + # transid (getLong) + # nodeid (getInt) + # commitsequence (getLong) + return results def fetch_result_set(self, statement): @@ -518,7 +586,7 @@ def putString(self, value): :type value: str """ - data = bytes(value) if isP2 else value.encode('utf-8') # type: ignore + data = value.encode('utf-8') # type: ignore length = len(data) if length < 40: self.__output.append(protocol.UTF8LEN0 + length) @@ -661,7 +729,7 @@ def putScaledTime(self, value): :type value: datetype.Time """ return self._putScaled(protocol.SCALEDTIMELEN0, - *datatype.TimeToTicks(value)) + *datatype.TimeToTicks(value,self.timezone_info)) def putScaledTimestamp(self, value): # type: (datatype.Timestamp) -> EncodedSession @@ -670,7 +738,7 @@ def putScaledTimestamp(self, value): :type value: datetime.datetime """ return self._putScaled(protocol.SCALEDTIMESTAMPLEN0, - *datatype.TimestampToTicks(value)) + *datatype.TimestampToTicks(value,self.timezone_info)) def putScaledDate(self, value): # type: (datatype.Date) -> EncodedSession @@ -897,7 +965,28 @@ def getClob(self): raise DataError('Not a clob') + + __shifters = [1,10,100,1000,10000,100000,1000000,10000000,100000000, 1000000000] + + def __unpack(self,scale,time): + shiftr= self.__shifters[scale] + ticks = time // shiftr + fraction = time % shiftr + if scale > 6: + micros = fraction // self.__shifters[scale-6] + else: + micros = fraction * self.__shifters[6-scale] + if micros < 0: + micros %= 1000000 + ticks += 1 + return (ticks,micros) + + # def __unpack(self,scale,time): + # ticks = decimal.Decimal(time) / decimal.Decimal(10**scale) + # return (round(int(ticks)), int((ticks % 1) * decimal.Decimal(1000000))) + def getScaledTime(self): + # type: () -> datatype.Time """Read the next Scaled Time value off the session. @@ -908,9 +997,8 @@ def getScaledTime(self): if code >= protocol.SCALEDTIMELEN1 and code <= protocol.SCALEDTIMELEN8: scale = fromByteString(self._takeBytes(1)) time = fromSignedByteString(self._takeBytes(code - protocol.SCALEDTIMELEN0)) - ticks = decimal.Decimal(time) / decimal.Decimal(10**scale) - return datatype.TimeFromTicks(round(int(ticks)), - int((ticks % 1) * decimal.Decimal(1000000))) + seconds,micros = self.__unpack(scale,time) + return datatype.TimeFromTicks(seconds,micros,self.timezone_info) raise DataError('Not a scaled time') @@ -925,9 +1013,8 @@ def getScaledTimestamp(self): if code >= protocol.SCALEDTIMESTAMPLEN1 and code <= protocol.SCALEDTIMESTAMPLEN8: scale = fromByteString(self._takeBytes(1)) stamp = fromSignedByteString(self._takeBytes(code - protocol.SCALEDTIMESTAMPLEN0)) - ticks = decimal.Decimal(stamp) / decimal.Decimal(10**scale) - return datatype.TimestampFromTicks(round(int(ticks)), - int((ticks % 1) * decimal.Decimal(1000000))) + seconds,micros = self.__unpack(scale,stamp) + return datatype.TimestampFromTicks(seconds,micros,self.timezone_info) raise DataError('Not a scaled timestamp') @@ -942,7 +1029,7 @@ def getScaledDate(self): if code >= protocol.SCALEDDATELEN1 and code <= protocol.SCALEDDATELEN8: scale = fromByteString(self._takeBytes(1)) date = fromSignedByteString(self._takeBytes(code - protocol.SCALEDDATELEN0)) - return datatype.DateFromTicks(round(date / 10.0 ** scale)) + return datatype.DateFromTicks(date//(10**scale)) raise DataError('Not a scaled date') diff --git a/requirements.txt b/requirements.txt index 432c719..67dfed8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -pytz>=2015.4 +tzlocal>=3.0 + diff --git a/setup.py b/setup.py index 4955f52..908f661 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ url='https://github.com/nuodb/nuodb-python', license='BSD License', long_description=open(readme).read(), - install_requires=['pytz>=2015.4', 'ipaddress'], + install_requires=['tzlocal>=3.0', 'ipaddress'], extras_require=dict(crypto='cryptography>=2.6.1'), classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/mock_tzs.py b/tests/mock_tzs.py index d149f9b..974ead2 100644 --- a/tests/mock_tzs.py +++ b/tests/mock_tzs.py @@ -2,21 +2,16 @@ # -*- coding: utf-8 -*- from datetime import tzinfo -from datetime import datetime +from datetime import datetime, timezone import os - -import pytz +import zoneinfo +import tzlocal import pynuodb -if os.path.exists('/etc/timezone'): - with open('/etc/timezone') as tzf: - Local = pytz.timezone(tzf.read().strip()) -else: - with open('/etc/localtime', 'rb') as tlf: - Local = pytz.build_tzinfo('localtime', tlf) # type: ignore - -UTC = pytz.timezone('UTC') +Local = tzlocal.get_localzone() + +UTC = timezone.utc class _MyOffset(tzinfo): @@ -27,7 +22,7 @@ class _MyOffset(tzinfo): This class can be used to do exactly the same thing to the test val. ''' def utcoffset(self, dt): - return Local.localize(datetime.now()).utcoffset() + return datetime.now().utcoffset() MyOffset = _MyOffset()