Commit fd737b5f authored by Jérome Perrin's avatar Jérome Perrin

Compatibility with DateTime >= 3

On the way to Zope 4, the biggest change was DateTime >= 3, which has these main differences:

 - `DateTime` instances are new style classes
 - `DateTime.__eq__` now take into the timezone into account
 - `DateTime` supports timezone naive ( https://docs.python.org/3/library/datetime.html#aware-and-naive-objects ) - actually this is since DateTime 2.12 , but since we fully patched the method where the parsing happens, ERP5's patched DateTime never used timezone naive dates.
 - pickles are also different.

These are changes to prepare ERP5 code base to support DateTime >= 3, while keeping compatibility with the previous behavior, this means that we keep patching to keep the same behavior as DateTime 2, so that `DateTime.__eq__` ignores timezone difference and `DateTime` does not support timezone naive dates - these two different behavior seemed to introduce too much breakages, especially in project code, with no immediate benefit for our usages. For now, while ERP5 is still based on Zope 2 / DateTime 2 the fact that DateTime are new style classes has no impact, this will only impact once we switch to Zope 4.

This also change the code to adjust the part where we were depending on implementation details of DateTime and where the details changed on DateTime 3, sometimes exposing higher level API, like the new `timeZoneContext` to use in tests.



See merge request !1593
parents 0243d5c8 6083a15b
Pipeline #26099 failed with stage
in 0 seconds
......@@ -27,12 +27,16 @@
#
##############################################################################
import os
import unittest
import zodbpickle.fastpickle as pickle
from DateTime import DateTime
from erp5.component.module.DateUtils import addToDate, getIntervalListBetweenDates, \
atTheEndOfPeriod, getClosestDate
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.utils import timeZoneContext
class TestDateUtils(unittest.TestCase):
"""
......@@ -199,8 +203,141 @@ class TestPinDateTime(ERP5TypeTestCase):
self.assertGreaterEqual(DateTime(), actual_begin_date)
class TestTimeZoneContext(ERP5TypeTestCase):
def afterSetUp(self):
self.reference_date_in_utc = DateTime('2001/02/03 00:00:00 UTC')
self.actual_timezone = DateTime().timezone()
self.actual_environ_tz = os.environ.get('TZ')
def test_timezone_context_UTC(self):
with timeZoneContext('UTC'):
self.assertEqual(DateTime().timezone(), 'UTC')
self.assertEqual(
DateTime(2001, 2, 3).toZone('UTC'), self.reference_date_in_utc)
self.assertEqual(DateTime().timezone(), self.actual_timezone)
self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
def test_timezone_context_with_dst(self):
with timeZoneContext('Europe/Paris'):
self.assertEqual(DateTime(2021, 2, 1).timezone(), 'CET')
self.assertEqual(DateTime(2021, 7, 1).timezone(), 'CEST')
self.assertEqual(
DateTime(2001, 2, 3, 1, 0, 0).toZone('UTC'),
self.reference_date_in_utc)
self.assertEqual(DateTime().timezone(), self.actual_timezone)
self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
def test_timezone_context_without_dst(self):
with timeZoneContext('Asia/Tokyo'):
self.assertEqual(DateTime().timezone(), 'JST')
self.assertEqual(
DateTime(2001, 2, 3, 9, 0, 0).toZone('UTC'), self.reference_date_in_utc)
self.assertEqual(DateTime().timezone(), self.actual_timezone)
self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
def test_timezone_abbreviation(self):
with timeZoneContext('GMT-7'):
self.assertEqual(DateTime(2021, 2, 1).timezone(), 'GMT-7')
self.assertEqual(DateTime(2021, 7, 1).timezone(), 'GMT-7')
self.assertEqual(
DateTime(2001, 2, 2, 17, 0, 0).toZone('UTC'), self.reference_date_in_utc)
self.assertEqual(DateTime().timezone(), self.actual_timezone)
self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
class TestDateTimePatch(ERP5TypeTestCase):
"""Tests for monkey patches in Products.ERP5Type.patches.DateTimePatch
"""
def _test_pickle(self, dt, data):
"""Assert pickle `data` when loaded is equal to DateTime `dt`
"""
new = pickle.loads(data)
if hasattr(DateTime, '__slots__'):
for key in DateTime.__slots__:
self.assertEqual(getattr(dt, key), getattr(new, key))
else:
# BBB DateTime 2
self.assertEqual(dt.__dict__, new.__dict__)
# pickles from "current" ERP5
# around commit fcaa5dddbd (Zelenium: update html2canvas to version 1.4.1, 2022-04-18)
def test_pickle_europe_paris(self):
dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xba\xb1\x00\x00\x00U\x0cEurope/Parisq\x03tb.'
self._test_pickle(dt, data)
def test_pickle_UTC(self):
dt = DateTime('2001/02/03 04:05:06 UTC')
data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xc1\xb9\x00\x00\x00U\x03UTCq\x03tb.'
self._test_pickle(dt, data)
# "r15569" was an old patch to DateTime.__getstate__ that we keep compatibility with.
# It was a svn commit that was convert to git commit 7b89b86838 (Tweak DateTime pickle
# representation to avoid using 370 bytes per DateTime, but ~80 bytes instead.
# Retain backward compatibility with regular DateTime default serialisation., 2007-08-08)
def test_pickle_europe_paris_r15569(self):
dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U 2001/02/03 04:05:06 Europe/Parissb.'
self._test_pickle(dt, data)
def test_pickle_UTC_r15569(self):
dt = DateTime('2001/02/03 04:05:06 UTC')
data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U\x172001/02/03 04:05:06 UTCsb.'
self._test_pickle(dt, data)
def test_pickle_protocol_3(self):
dt = DateTime()
data = pickle.dumps(dt, 3)
self._test_pickle(dt, data)
def test_pickle_dumps_loads(self):
for i in (
'2007/01/02 12:34:56.789',
'2007/01/02 12:34:56.789 GMT+0200',
'2007/01/02 12:34:56.789 JST',
'2007/01/02 12:34:56.789 +0300',
'2007/01/02 12:34:56.789 +0430',
'2007/01/02 12:34:56.789 +1237',
):
dt = DateTime(i)
self._test_pickle(dt, pickle.dumps(dt, 1))
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestDateUtils))
suite.addTest(unittest.makeSuite(TestPinDateTime))
suite.addTest(unittest.makeSuite(TestTimeZoneContext))
suite.addTest(unittest.makeSuite(TestDateTimePatch))
# also run original tests from DateTime module
# pylint:disable=no-name-in-module
try:
import DateTime.tests.testDateTime as test_datetime
except ImportError:
from DateTime.tests import test_datetime
# pylint:enable=no-name-in-module
class DateTimeTests(test_datetime.DateTimeTests):
testTimezoneNaiveHandling = unittest.expectedFailure(
test_datetime.DateTimeTests.testTimezoneNaiveHandling)
# This test is only in DateTime >= 3
if hasattr(test_datetime.DateTimeTests, 'test_intl_format_hyphen'):
test_intl_format_hyphen = unittest.expectedFailure(
test_datetime.DateTimeTests.test_intl_format_hyphen)
# These 3 tests are only in DateTime 2
if hasattr(test_datetime.DateTimeTests, 'test_pickle_new_with_micros'):
test_pickle_new_with_micros = unittest.expectedFailure(
test_datetime.DateTimeTests.test_pickle_new_with_micros)
if hasattr(test_datetime.DateTimeTests, 'test_pickle_new_with_tz'):
test_pickle_new_with_tz = unittest.expectedFailure(
test_datetime.DateTimeTests.test_pickle_new_with_tz)
if hasattr(test_datetime.DateTimeTests, 'testLegacyTimezones'):
testLegacyTimezones = unittest.expectedFailure(
test_datetime.DateTimeTests.testLegacyTimezones)
suite.addTest(unittest.makeSuite(DateTimeTests))
return suite
import os, time
from DateTime import DateTime
from Products.ERP5Type.tests.utils import timeZoneContext
current_timezone_contexts = []
def setTimezone(timezone):
# timezone must be for example GMT-7
os.environ['TZ'] = timezone
time.tzset()
DateTime._isDST = False
DateTime._localzone = DateTime._localzone0 = DateTime._localzone1 = timezone
"""Change the default timezone to `timezone`.
"""
if current_timezone_contexts:
resetTimeZone()
tzc = timeZoneContext(timezone)
tzc.__enter__()
current_timezone_contexts.append(tzc)
return "Timezone Updated"
def resetTimeZone():
"""Reset the timezone that might have been set by `setTimezone`
"""
current_timezone_contexts.pop().__exit__(None, None, None)
......@@ -27,6 +27,8 @@
#
##############################################################################
import calendar
from DateTime import DateTime
from AccessControl import ClassSecurityInfo
from Products.ERP5Type.Globals import InitializeClass
......@@ -228,13 +230,14 @@ class PeriodicityMixin:
next_start_date = next_start_date.toZone(timezone)
return next_start_date
# XXX May be we should create a Date class for following methods ???
security.declareProtected(Permissions.AccessContentsInformation, 'getWeekDayList')
def getWeekDayList(self):
"""
returns something like ['Sunday','Monday',...]
"""
return DateTime._days
return [
calendar.day_name[i]
for i in calendar.Calendar(calendar.SUNDAY).iterweekdays()]
security.declareProtected(Permissions.AccessContentsInformation, 'getWeekDayItemList')
def getWeekDayItemList(self):
......@@ -249,9 +252,10 @@ class PeriodicityMixin:
"""
returns something like [('January', 1), ('February', 2),...]
"""
# DateTime._months return '' as first item
return [(Message(domain='erp5_ui', message=DateTime._months[i]), i) \
for i in range(1, len(DateTime._months))]
# calendar.month_name return '' as first item
month_name = list(calendar.month_name)
return [(Message(domain='erp5_ui', message=month_name[i]), i) \
for i in range(1, len(month_name))]
security.declareProtected(Permissions.AccessContentsInformation,'getPeriodicityWeekDayList')
def getPeriodicityWeekDayList(self):
......
......@@ -241,7 +241,7 @@ class TestOOoImport(TestOOoImportMixin):
sorted(['male' for i in range(num)]),
sorted([person_list[i].getGender() for i in range(num)]))
self.assertEqual(
sorted([DateTime('2008/02/%02d %s' % (i+1, 'GMT')) for i in range(num)]),
sorted([DateTime('2008-02-%02d' % (i+1)) for i in range(num)]),
sorted([person_list[i].getStartDate() for i in range(num)]))
def stepCheckImportFloatsAndPercentage(self, sequence=None, sequence_list=None, **kw):
......
......@@ -35,10 +35,29 @@ SyntaxError, DateError, TimeError, localtime, time
STATE_KEY = 'str'
# DateTime 3 changed the __eq__ behavior and d1 == d2 only if they have the same same
# timezone. With DateTime 2 two dates from different timezones representing the same
# time were equal. This patch keeps the behavior from DateTime 2.
# See zopefoundation/DateTime commit fff6d04 (Various cleanups, improve unpickling
# speed and distinguish between equal representations and references to equal points
# in time., 2011-05-06)
DateTimeKlass.__eq__ = DateTimeKlass.equalTo
# ERP5 Patch for different pickle implementation, to optimize for disk usage.
# We had different __getstate__ implementations, so we need __setstate__ to support
# loading these formats that might be present in ZODBs.
# This patch does not have support for timezone naive flag, because we don't need it
# so far and also probably because we did not notice that it was added in original
# DateTime.
original_DateTime__setstate__ = DateTimeKlass.__setstate__
def DateTime__setstate__(self, state):
self.__dict__.clear()
try: # BBB DateTime 2.12.8
self.__dict__.clear()
except AttributeError:
pass
self._timezone_naive = False
if isinstance(state, tuple):
t, tz = state
ms = (t - math.floor(t))
......@@ -60,6 +79,13 @@ def DateTime__getstate__(self):
DateTimeKlass.__getstate__ = DateTime__getstate__
# ERP5 Patch to have different parsing rules.
# We have a patch since e0eba4791a (Authorised date manipulation before
# year 1000, 2008-01-28), which replaced the method with an implementation
# that did not change since, so we don't have new behaviors of DateTime
# the most visible change might be that we don't have "timezone naive"
# support.
def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
# Parse date-time components from a string
month=year=tz=tm=None
......@@ -82,6 +108,9 @@ def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
else: tz = None # Decide later, since the default time zone
# could depend on the date.
# XXX we don't support timezone naive in this patch
self._timezone_naive = False
ints,dels=[],[]
i,l=0,len(st)
while i < l:
......@@ -249,18 +278,11 @@ def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
DateTimeKlass._parse = DateTime_parse
if __name__ == '__main__':
for i in ('2007/01/02 12:34:56.789',
'2007/01/02 12:34:56.789 GMT+0200',
'2007/01/02 12:34:56.789 JST',
'2007/01/02 12:34:56.789 +0300',
'2007/01/02 12:34:56.789 +0430',
'2007/01/02 12:34:56.789 +1237',
):
a = DateTimeKlass(i)
b = DateTimeKlass()
b.__setstate__(a.__getstate__())
print(a, a.__dict__ == b.__dict__)
for i in a.__dict__.keys():
if a.__dict__[i] != b.__dict__[i]:
print(i, a.__dict__[i], b.__dict__[i])
# DateTime 3 removed exceptions as class attributes (since
# zopefoundation/DateTime commit 8114618 ), but we have some code expecting
# these attributes, so undo this patch for convenience.
DateTimeKlass.DateTimeError = DateTimeError
DateTimeKlass.SyntaxError = SyntaxError
DateTimeKlass.DateError = DateError
DateTimeKlass.TimeError = TimeError
......@@ -66,6 +66,7 @@ from Products.ERP5Type.Utils import convertToUpperCase, str2bytes
from Products.ERP5Type.tests.backportUnittest import SetupSiteError
from Products.ERP5Type.tests.utils import addUserToDeveloperRole
from Products.ERP5Type.tests.utils import parseListeningAddress
from Products.ERP5Type.tests.utils import timeZoneContext
# Quiet messages when installing business templates
install_bt5_quiet = 0
......@@ -376,15 +377,10 @@ class ERP5TypeTestCaseMixin(ProcessingNodeTestCase, PortalTestCase):
self.pinDateTime(None)
def setTimeZoneToUTC(self):
# Make sure tests runs with UTC timezone. Some tests are checking values
# based on now, and this could give unexpected results:
# DateTime("2016/10/31") - DateTime("2016/10/30") = 1.0416666666666667 if
# you are running on a timezone like Europe/Paris, while it return 1.0 for
# UTC
os.environ['TZ'] = "UTC"
time.tzset()
DateTime._isDST = False
DateTime._localzone = DateTime._localzone0 = DateTime._localzone1 = "UTC"
# Deprecated, prefer using `timeZoneContext` context manager instead.
timezone = timeZoneContext('UTC')
timezone.__enter__()
self.addCleanup(timezone.__exit__, None, None, None)
def getDefaultSystemPreference(self):
id = 'default_system_preference'
......
......@@ -28,12 +28,16 @@
"""Utility functions and classes for unit testing
"""
import contextlib
from datetime import datetime
import errno
import os
import logging
import mock
import random
import socket
import sys
import time
import unittest
import ZODB
import zLOG
......@@ -49,6 +53,7 @@ from email import message_from_string
from Products.ERP5Type.Globals import PersistentMapping
from Products.ERP5Type.Utils import simple_decorator
from Products.ZSQLCatalog.SQLCatalog import Catalog
import pytz
import six
class FileUpload(file):
......@@ -707,3 +712,35 @@ class SubcontentReindexingWrapper(object):
self.assertEqual(expected_path_set, catalogged_object_path_set)
finally:
Catalog.catalogObjectList = orig_catalogObjectList
@contextlib.contextmanager
def timeZoneContext(timezone):
"""Context manager to change timezone in tests.
"""
saved_TZ = os.environ.get('TZ')
os.environ['TZ'] = timezone
time.tzset()
if timezone in pytz.all_timezones:
_multipleZones = time.daylight
_localzone0 = time.tzname[0]
_localzone1 = time.tzname[1] if time.daylight else time.tzname[0]
else:
_multipleZones = False
_localzone0 = _localzone1 = timezone
if hasattr(sys.modules['DateTime.DateTime'].DateTime, '_localzone0'):
patch_target = sys.modules['DateTime.DateTime'].DateTime
else:
# BBB DateTime 2
patch_target = sys.modules['DateTime.DateTime']
try:
with mock.patch.object(patch_target, '_localzone0', new=_localzone0), \
mock.patch.object(patch_target, '_localzone1', new=_localzone1), \
mock.patch.object(patch_target, '_multipleZones', new=_multipleZones):
yield
finally:
os.environ.pop('TZ')
if saved_TZ:
os.environ['TZ'] = saved_TZ
time.tzset()
......@@ -31,18 +31,22 @@ from __future__ import absolute_import
import six
from six import string_types as basestring
import calendar
from .SearchKey import SearchKey
from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
from zLOG import LOG
from DateTime.DateTime import DateTime, DateTimeError, _cache
from DateTime.DateTime import DateTime, DateTimeError
from DateTime import Timezones
from Products.ZSQLCatalog.interfaces.search_key import ISearchKey
from zope.interface.verify import verifyClass
from Products.ZSQLCatalog.SearchText import parse
MARKER = []
timezone_dict = _cache._zmap
# We use standard DateTime timezone, with also some timezones that were
# included before and are used (at least) in the test suite.
timezone_set = set([tz.lower() for tz in Timezones()] + ['cet', 'cest'])
date_completion_format_dict = {
None: ['01/01/%s', '01/%s'],
......@@ -84,7 +88,7 @@ def castDate(value, change_timezone=True):
delimiter_count = countDelimiters(value)
if delimiter_count is not None and delimiter_count < 2:
split_value = value.split()
if split_value[-1].lower() in timezone_dict:
if split_value[-1].lower() in timezone_set:
value = '%s %s' % (date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (' '.join(split_value[:-1]), ), split_value[-1])
else:
value = date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (value, )
......@@ -106,7 +110,7 @@ def castDate(value, change_timezone=True):
delimiter_list = ' -/.:,+'
def getMonthLen(datetime):
return datetime._month_len[datetime.isLeapYear()][datetime.month()]
return calendar.monthrange(datetime.year(), datetime.month())[1]
def getYearLen(datetime):
return 365 + datetime.isLeapYear()
......@@ -119,7 +123,7 @@ def countDelimiters(value):
split_value = value.split()
if not split_value:
return None
if split_value[-1].lower() in timezone_dict:
if split_value[-1].lower() in timezone_set:
value = ' '.join(split_value[:-1])
# Count delimiters
delimiter_count = 0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment