Compatibility with DateTime >= 3

On the way to Zope 4, the biggest change was DateTime >= 3, which has these main differences:

 - `DateTime` instances are new style classes
 - `DateTime.__eq__` now take into the timezone into account
 - `DateTime` supports timezone naive ( https://docs.python.org/3/library/datetime.html#aware-and-naive-objects ) - actually this is since DateTime 2.12 , but since we fully patched the method where the parsing happens, ERP5's patched DateTime never used timezone naive dates.
 - pickles are also different.

These are changes to prepare ERP5 code base to support DateTime >= 3, while keeping compatibility with the previous behavior, this means that we keep patching to keep the same behavior as DateTime 2, so that `DateTime.__eq__` ignores timezone difference and `DateTime` does not support timezone naive dates - these two different behavior seemed to introduce too much breakages, especially in project code, with no immediate benefit for our usages. For now, while ERP5 is still based on Zope 2 / DateTime 2 the fact that DateTime are new style classes has no impact, this will only impact once we switch to Zope 4.

This also change the code to adjust the part where we were depending on implementation details of DateTime and where the details changed on DateTime 3, sometimes exposing higher level API, like the new `timeZoneContext` to use in tests.



See merge request nexedi/erp5!1593

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testDateUtils.py

@@ -27,12 +27,16 @@
 #
 ##############################################################################
 
+import os
 import unittest
 
+import zodbpickle.fastpickle as pickle
 from DateTime import DateTime
 from erp5.component.module.DateUtils import addToDate, getIntervalListBetweenDates, \
     atTheEndOfPeriod, getClosestDate
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+from Products.ERP5Type.tests.utils import timeZoneContext
+
 
 class TestDateUtils(unittest.TestCase):
   """
@@ -199,8 +203,141 @@ class TestPinDateTime(ERP5TypeTestCase):
     self.assertGreaterEqual(DateTime(), actual_begin_date)
 
 
+class TestTimeZoneContext(ERP5TypeTestCase):
+  def afterSetUp(self):
+    self.reference_date_in_utc = DateTime('2001/02/03 00:00:00 UTC')
+    self.actual_timezone = DateTime().timezone()
+    self.actual_environ_tz = os.environ.get('TZ')
+
+  def test_timezone_context_UTC(self):
+    with timeZoneContext('UTC'):
+      self.assertEqual(DateTime().timezone(), 'UTC')
+      self.assertEqual(
+        DateTime(2001, 2, 3).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_context_with_dst(self):
+    with timeZoneContext('Europe/Paris'):
+      self.assertEqual(DateTime(2021, 2, 1).timezone(), 'CET')
+      self.assertEqual(DateTime(2021, 7, 1).timezone(), 'CEST')
+      self.assertEqual(
+        DateTime(2001, 2, 3, 1, 0, 0).toZone('UTC'),
+        self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_context_without_dst(self):
+    with timeZoneContext('Asia/Tokyo'):
+      self.assertEqual(DateTime().timezone(), 'JST')
+      self.assertEqual(
+        DateTime(2001, 2, 3, 9, 0, 0).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_abbreviation(self):
+    with timeZoneContext('GMT-7'):
+      self.assertEqual(DateTime(2021, 2, 1).timezone(), 'GMT-7')
+      self.assertEqual(DateTime(2021, 7, 1).timezone(), 'GMT-7')
+      self.assertEqual(
+        DateTime(2001, 2, 2, 17, 0, 0).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+
+class TestDateTimePatch(ERP5TypeTestCase):
+  """Tests for monkey patches in Products.ERP5Type.patches.DateTimePatch
+  """
+  def _test_pickle(self, dt, data):
+    """Assert pickle `data` when loaded is equal to DateTime `dt`
+    """
+    new = pickle.loads(data)
+    if hasattr(DateTime, '__slots__'):
+      for key in DateTime.__slots__:
+        self.assertEqual(getattr(dt, key), getattr(new, key))
+    else:
+      # BBB DateTime 2
+      self.assertEqual(dt.__dict__, new.__dict__)
+
+  # pickles from "current" ERP5
+  # around commit fcaa5dddbd (Zelenium: update html2canvas to version 1.4.1, 2022-04-18)
+  def test_pickle_europe_paris(self):
+    dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xba\xb1\x00\x00\x00U\x0cEurope/Parisq\x03tb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_UTC(self):
+    dt = DateTime('2001/02/03 04:05:06 UTC')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xc1\xb9\x00\x00\x00U\x03UTCq\x03tb.'
+    self._test_pickle(dt, data)
+
+  # "r15569" was an old patch to DateTime.__getstate__ that we keep compatibility with.
+  # It was a svn commit that was convert to git commit 7b89b86838 (Tweak DateTime pickle
+  # representation to avoid using 370 bytes per DateTime, but ~80 bytes instead.
+  # Retain backward compatibility with regular DateTime default serialisation., 2007-08-08)
+  def test_pickle_europe_paris_r15569(self):
+    dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U 2001/02/03 04:05:06 Europe/Parissb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_UTC_r15569(self):
+    dt = DateTime('2001/02/03 04:05:06 UTC')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U\x172001/02/03 04:05:06 UTCsb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_protocol_3(self):
+    dt = DateTime()
+    data = pickle.dumps(dt, 3)
+    self._test_pickle(dt, data)
+
+  def test_pickle_dumps_loads(self):
+    for i in (
+      '2007/01/02 12:34:56.789',
+      '2007/01/02 12:34:56.789 GMT+0200',
+      '2007/01/02 12:34:56.789 JST',
+      '2007/01/02 12:34:56.789 +0300',
+      '2007/01/02 12:34:56.789 +0430',
+      '2007/01/02 12:34:56.789 +1237',
+    ):
+      dt = DateTime(i)
+      self._test_pickle(dt, pickle.dumps(dt, 1))
+
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestDateUtils))
   suite.addTest(unittest.makeSuite(TestPinDateTime))
+  suite.addTest(unittest.makeSuite(TestTimeZoneContext))
+  suite.addTest(unittest.makeSuite(TestDateTimePatch))
+
+  # also run original tests from DateTime module
+  # pylint:disable=no-name-in-module
+  try:
+    import DateTime.tests.testDateTime as test_datetime
+  except ImportError:
+    from DateTime.tests import test_datetime
+  # pylint:enable=no-name-in-module
+
+  class DateTimeTests(test_datetime.DateTimeTests):
+    testTimezoneNaiveHandling = unittest.expectedFailure(
+      test_datetime.DateTimeTests.testTimezoneNaiveHandling)
+
+    # This test is only in DateTime >= 3
+    if hasattr(test_datetime.DateTimeTests, 'test_intl_format_hyphen'):
+      test_intl_format_hyphen = unittest.expectedFailure(
+        test_datetime.DateTimeTests.test_intl_format_hyphen)
+
+    # These 3 tests are only in DateTime 2
+    if hasattr(test_datetime.DateTimeTests, 'test_pickle_new_with_micros'):
+      test_pickle_new_with_micros = unittest.expectedFailure(
+        test_datetime.DateTimeTests.test_pickle_new_with_micros)
+    if hasattr(test_datetime.DateTimeTests, 'test_pickle_new_with_tz'):
+      test_pickle_new_with_tz = unittest.expectedFailure(
+        test_datetime.DateTimeTests.test_pickle_new_with_tz)
+    if hasattr(test_datetime.DateTimeTests, 'testLegacyTimezones'):
+      testLegacyTimezones = unittest.expectedFailure(
+        test_datetime.DateTimeTests.testLegacyTimezones)
+
+  suite.addTest(unittest.makeSuite(DateTimeTests))
+
   return suite

bt5/erp5_web_renderjs_ui_test/ExtensionTemplateItem/portal_components/extension.erp5.WebRenderJSTestUtils.py

-import os, time
-from DateTime import DateTime
+from Products.ERP5Type.tests.utils import timeZoneContext
+
+current_timezone_contexts = []
 
 def setTimezone(timezone):
-  # timezone must be for example GMT-7
-  os.environ['TZ'] = timezone
-  time.tzset()
-  DateTime._isDST = False
-  DateTime._localzone = DateTime._localzone0 = DateTime._localzone1 = timezone
+  """Change the default timezone to `timezone`.
+  """
+  if current_timezone_contexts:
+    resetTimeZone()
+
+  tzc = timeZoneContext(timezone)
+  tzc.__enter__()
+  current_timezone_contexts.append(tzc)
+
   return "Timezone Updated"
+
+
+def resetTimeZone():
+  """Reset the timezone that might have been set by `setTimezone`
+  """
+  current_timezone_contexts.pop().__exit__(None, None, None)

product/ERP5/mixin/periodicity.py

@@ -27,6 +27,8 @@
 #
 ##############################################################################
 
+import calendar
+
 from DateTime import DateTime
 from AccessControl import ClassSecurityInfo
 from Products.ERP5Type.Globals import InitializeClass
@@ -228,13 +230,14 @@ class PeriodicityMixin:
           next_start_date = next_start_date.toZone(timezone)
         return next_start_date
 
-  # XXX May be we should create a Date class for following methods ???
   security.declareProtected(Permissions.AccessContentsInformation, 'getWeekDayList')
   def getWeekDayList(self):
     """
     returns something like ['Sunday','Monday',...]
     """
-    return DateTime._days
+    return [
+      calendar.day_name[i]
+      for i in calendar.Calendar(calendar.SUNDAY).iterweekdays()]
 
   security.declareProtected(Permissions.AccessContentsInformation, 'getWeekDayItemList')
   def getWeekDayItemList(self):
@@ -249,9 +252,10 @@ class PeriodicityMixin:
     """
     returns something like [('January', 1), ('February', 2),...]
     """
-    # DateTime._months return '' as first item
-    return [(Message(domain='erp5_ui', message=DateTime._months[i]), i) \
-            for i in range(1, len(DateTime._months))]
+    # calendar.month_name return '' as first item
+    month_name = list(calendar.month_name)
+    return [(Message(domain='erp5_ui', message=month_name[i]), i) \
+            for i in range(1, len(month_name))]
 
   security.declareProtected(Permissions.AccessContentsInformation,'getPeriodicityWeekDayList')
   def getPeriodicityWeekDayList(self):

product/ERP5OOo/tests/testOOoImport.py

@@ -241,7 +241,7 @@ class TestOOoImport(TestOOoImportMixin):
       sorted(['male' for i in range(num)]),
       sorted([person_list[i].getGender() for i in range(num)]))
     self.assertEqual(
-      sorted([DateTime('2008/02/%02d %s' % (i+1, 'GMT')) for i in range(num)]),
+      sorted([DateTime('2008-02-%02d' % (i+1)) for i in range(num)]),
       sorted([person_list[i].getStartDate() for i in range(num)]))
 
   def stepCheckImportFloatsAndPercentage(self, sequence=None, sequence_list=None, **kw):

product/ERP5Type/patches/DateTimePatch.py

@@ -35,10 +35,29 @@ SyntaxError, DateError, TimeError, localtime, time
 
 STATE_KEY = 'str'
 
+# DateTime 3 changed the __eq__ behavior and d1 == d2 only if they have the same same
+# timezone. With DateTime 2 two dates from different timezones representing the same
+# time were equal. This patch keeps the behavior from DateTime 2.
+# See zopefoundation/DateTime commit fff6d04 (Various cleanups, improve unpickling
+# speed and distinguish between equal representations and references to equal points
+# in time., 2011-05-06)
+DateTimeKlass.__eq__ = DateTimeKlass.equalTo
+
+
+# ERP5 Patch for different pickle implementation, to optimize for disk usage.
+# We had different __getstate__ implementations, so we need __setstate__ to support
+# loading these formats that might be present in ZODBs.
+# This patch does not have support for timezone naive flag, because we don't need it
+# so far and also probably because we did not notice that it was added in original
+# DateTime.
 original_DateTime__setstate__ = DateTimeKlass.__setstate__
 
 def DateTime__setstate__(self, state):
+  try: # BBB DateTime 2.12.8
     self.__dict__.clear()
+  except AttributeError:
+    pass
+  self._timezone_naive = False
   if isinstance(state, tuple):
     t, tz = state
     ms = (t - math.floor(t))
@@ -60,6 +79,13 @@ def DateTime__getstate__(self):
 
 DateTimeKlass.__getstate__ = DateTime__getstate__
 
+
+# ERP5 Patch to have different parsing rules.
+# We have a patch since e0eba4791a (Authorised date manipulation before
+# year 1000, 2008-01-28), which replaced the method with an implementation
+# that did not change since, so we don't have new behaviors of DateTime
+# the most visible change might be that we don't have "timezone naive"
+# support.
 def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
   # Parse date-time components from a string
   month=year=tz=tm=None
@@ -82,6 +108,9 @@ def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
   else: tz = None  # Decide later, since the default time zone
   # could depend on the date.
 
+  # XXX we don't support timezone naive in this patch
+  self._timezone_naive = False
+
   ints,dels=[],[]
   i,l=0,len(st)
   while i < l:
@@ -249,18 +278,11 @@ def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
 
 DateTimeKlass._parse = DateTime_parse
 
-if __name__ == '__main__':
-  for i in ('2007/01/02 12:34:56.789',
-            '2007/01/02 12:34:56.789 GMT+0200',
-            '2007/01/02 12:34:56.789 JST',
-            '2007/01/02 12:34:56.789 +0300',
-            '2007/01/02 12:34:56.789 +0430',
-            '2007/01/02 12:34:56.789 +1237',
-            ):
-    a = DateTimeKlass(i)
-    b = DateTimeKlass()
-    b.__setstate__(a.__getstate__())
-    print(a, a.__dict__ == b.__dict__)
-    for i in a.__dict__.keys():
-      if a.__dict__[i] != b.__dict__[i]:
-        print(i, a.__dict__[i], b.__dict__[i])
+
+# DateTime 3 removed exceptions as class attributes (since
+# zopefoundation/DateTime commit 8114618 ), but we have some code expecting
+# these attributes, so undo this patch for convenience.
+DateTimeKlass.DateTimeError = DateTimeError
+DateTimeKlass.SyntaxError = SyntaxError
+DateTimeKlass.DateError = DateError
+DateTimeKlass.TimeError = TimeError

product/ERP5Type/tests/ERP5TypeTestCase.py

@@ -66,6 +66,7 @@ from Products.ERP5Type.Utils import convertToUpperCase, str2bytes
 from Products.ERP5Type.tests.backportUnittest import SetupSiteError
 from Products.ERP5Type.tests.utils import addUserToDeveloperRole
 from Products.ERP5Type.tests.utils import parseListeningAddress
+from Products.ERP5Type.tests.utils import timeZoneContext
 
 # Quiet messages when installing business templates
 install_bt5_quiet = 0
@@ -376,15 +377,10 @@ class ERP5TypeTestCaseMixin(ProcessingNodeTestCase, PortalTestCase):
       self.pinDateTime(None)
 
     def setTimeZoneToUTC(self):
-      # Make sure tests runs with UTC timezone. Some tests are checking values
-      # based on now, and this could give unexpected results:
-      # DateTime("2016/10/31") - DateTime("2016/10/30") = 1.0416666666666667 if
-      # you are running on a timezone like Europe/Paris, while it return 1.0 for
-      # UTC
-      os.environ['TZ'] = "UTC"
-      time.tzset()
-      DateTime._isDST = False
-      DateTime._localzone = DateTime._localzone0 = DateTime._localzone1 = "UTC"
+      # Deprecated, prefer using `timeZoneContext` context manager instead.
+      timezone = timeZoneContext('UTC')
+      timezone.__enter__()
+      self.addCleanup(timezone.__exit__, None, None, None)
 
     def getDefaultSystemPreference(self):
       id = 'default_system_preference'

product/ERP5Type/tests/utils.py

@@ -28,12 +28,16 @@
 
 """Utility functions and classes for unit testing
 """
+import contextlib
+from datetime import datetime
 import errno
 import os
 import logging
+import mock
 import random
 import socket
 import sys
+import time
 import unittest
 import ZODB
 import zLOG
@@ -49,6 +53,7 @@ from email import message_from_string
 from Products.ERP5Type.Globals import PersistentMapping
 from Products.ERP5Type.Utils import simple_decorator
 from Products.ZSQLCatalog.SQLCatalog import Catalog
+import pytz
 import six
 
 class FileUpload(file):
@@ -707,3 +712,35 @@ class SubcontentReindexingWrapper(object):
       self.assertEqual(expected_path_set, catalogged_object_path_set)
     finally:
       Catalog.catalogObjectList = orig_catalogObjectList
+
+
+@contextlib.contextmanager
+def timeZoneContext(timezone):
+  """Context manager to change timezone in tests.
+  """
+  saved_TZ = os.environ.get('TZ')
+  os.environ['TZ'] = timezone
+  time.tzset()
+  if timezone in pytz.all_timezones:
+    _multipleZones = time.daylight
+    _localzone0 = time.tzname[0]
+    _localzone1 = time.tzname[1] if time.daylight else time.tzname[0]
+  else:
+    _multipleZones = False
+    _localzone0 = _localzone1 = timezone
+  if hasattr(sys.modules['DateTime.DateTime'].DateTime, '_localzone0'):
+    patch_target = sys.modules['DateTime.DateTime'].DateTime
+  else:
+    # BBB DateTime 2
+    patch_target = sys.modules['DateTime.DateTime']
+
+  try:
+    with mock.patch.object(patch_target, '_localzone0', new=_localzone0), \
+        mock.patch.object(patch_target, '_localzone1', new=_localzone1), \
+        mock.patch.object(patch_target, '_multipleZones', new=_multipleZones):
+      yield
+  finally:
+    os.environ.pop('TZ')
+    if saved_TZ:
+      os.environ['TZ'] = saved_TZ
+    time.tzset()

product/ZSQLCatalog/SearchKey/DateTimeKey.py

@@ -31,18 +31,22 @@ from __future__ import absolute_import
 
 import six
 from six import string_types as basestring
+import calendar
 from .SearchKey import SearchKey
 from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
 from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
 from zLOG import LOG
-from DateTime.DateTime import DateTime, DateTimeError, _cache
+from DateTime.DateTime import DateTime, DateTimeError
+from DateTime import Timezones
 from Products.ZSQLCatalog.interfaces.search_key import ISearchKey
 from zope.interface.verify import verifyClass
 from Products.ZSQLCatalog.SearchText import parse
 
 MARKER = []
 
-timezone_dict = _cache._zmap
+# We use standard DateTime timezone, with also some timezones that were
+# included before and are used (at least) in the test suite.
+timezone_set = set([tz.lower() for tz in Timezones()] + ['cet', 'cest'])
 
 date_completion_format_dict = {
   None: ['01/01/%s', '01/%s'],
@@ -84,7 +88,7 @@ def castDate(value, change_timezone=True):
       delimiter_count = countDelimiters(value)
       if delimiter_count is not None and delimiter_count < 2:
         split_value = value.split()
-        if split_value[-1].lower() in timezone_dict:
+        if split_value[-1].lower() in timezone_set:
           value = '%s %s' % (date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (' '.join(split_value[:-1]), ), split_value[-1])
         else:
           value = date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (value, )
@@ -106,7 +110,7 @@ def castDate(value, change_timezone=True):
 delimiter_list = ' -/.:,+'
 
 def getMonthLen(datetime):
-  return datetime._month_len[datetime.isLeapYear()][datetime.month()]
+  return calendar.monthrange(datetime.year(), datetime.month())[1]
 
 def getYearLen(datetime):
   return 365 + datetime.isLeapYear()
@@ -119,7 +123,7 @@ def countDelimiters(value):
   split_value = value.split()
   if not split_value:
     return None
-  if split_value[-1].lower() in timezone_dict:
+  if split_value[-1].lower() in timezone_set:
     value = ' '.join(split_value[:-1])
   # Count delimiters
   delimiter_count = 0