*: compatibility with DateTime >= 3 🚧

~~drop the patch in DateTime._parse_args which cause DateTime to
support timezone naive dates~~
~~adjust __setstate__ to pickle the timezone naive flag~~

=> no we keep it the same patch as it was, without timezone
native support

adjust code relying on DateTime private attributes

adjust tests for new DateTime.__eq__ behavior (instances with
different timezones are different)

more testing, especially for various __setstate__ versions

introduce a context manager to change timezone in tests.

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testDateUtils.py

@@ -27,13 +27,16 @@
 #
 ##############################################################################
 
+import os
 import unittest
 
+import zodbpickle.fastpickle as pickle
 from DateTime import DateTime
 from erp5.component.module.DateUtils import addToDate, getIntervalListBetweenDates, \
-    atTheEndOfPeriod, getClosestDate
+    atTheEndOfPeriod, getClosestDate, timeZoneContext
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 
+
 class TestDateUtils(unittest.TestCase):
   """
   Tests DateUtils
@@ -199,8 +202,129 @@ class TestPinDateTime(ERP5TypeTestCase):
     self.assertGreaterEqual(DateTime(), actual_begin_date)
 
 
+class TestTimeZoneContext(ERP5TypeTestCase):
+  def afterSetUp(self):
+    self.reference_date_in_utc = DateTime('2001/02/03 00:00:00 UTC')
+    self.actual_timezone = DateTime().timezone()
+    self.actual_environ_tz = os.environ.get('TZ')
+
+  def test_timezone_context_UTC(self):
+    with timeZoneContext('UTC'):
+      self.assertEqual(DateTime().timezone(), 'UTC')
+      self.assertEqual(
+        DateTime(2001, 2, 3).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_context_with_dst(self):
+    with timeZoneContext('Europe/Paris'):
+      self.assertEqual(DateTime(2021, 2, 1).timezone(), 'CET')
+      self.assertEqual(DateTime(2021, 7, 1).timezone(), 'CEST')
+      self.assertEqual(
+        DateTime(2001, 2, 3, 1, 0, 0).toZone('UTC'),
+        self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_context_without_dst(self):
+    with timeZoneContext('Asia/Tokyo'):
+      self.assertEqual(DateTime().timezone(), 'JST')
+      self.assertEqual(
+        DateTime(2001, 2, 3, 9, 0, 0).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_abbreviation(self):
+    with timeZoneContext('GMT-7'):
+      self.assertEqual(DateTime(2021, 2, 1).timezone(), 'GMT-7')
+      self.assertEqual(DateTime(2021, 7, 1).timezone(), 'GMT-7')
+      self.assertEqual(
+        DateTime(2001, 2, 2, 17, 0, 0).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+
+class TestDateTimePatch(ERP5TypeTestCase):
+  """Tests for monkey patches in Products.ERP5Type.patches.DateTimePatch
+  """
+  def _test_pickle(self, dt, data):
+    """Assert pickle `data` when loaded is equal to DateTime `dt`
+    """
+    new = pickle.loads(data)
+    if hasattr(DateTime, '__slots__'):
+      for key in DateTime.__slots__:
+        self.assertEqual(getattr(dt, key), getattr(new, key))
+    else:
+      # BBB DateTime 2
+      self.assertEqual(dt.__dict__, new.__dict__)
+
+  # pickles from "current" ERP5
+  # around commit fcaa5dddbd (Zelenium: update html2canvas to version 1.4.1, 2022-04-18)
+  def test_pickle_europe_paris(self):
+    dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xba\xb1\x00\x00\x00U\x0cEurope/Parisq\x03tb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_UTC(self):
+    dt = DateTime('2001/02/03 04:05:06 UTC')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xc1\xb9\x00\x00\x00U\x03UTCq\x03tb.'
+    self._test_pickle(dt, data)
+
+  # "r15569" was an old patch to DateTime.__getstate__ that we keep compatibility with.
+  # It was a svn commit that was convert to git commit 7b89b86838 (Tweak DateTime pickle
+  # representation to avoid using 370 bytes per DateTime, but ~80 bytes instead.
+  # Retain backward compatibility with regular DateTime default serialisation., 2007-08-08)
+  def test_pickle_europe_paris_r15569(self):
+    dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U 2001/02/03 04:05:06 Europe/Parissb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_UTC_r15569(self):
+    dt = DateTime('2001/02/03 04:05:06 UTC')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U\x172001/02/03 04:05:06 UTCsb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_protocol_3(self):
+    dt = DateTime()
+    data = pickle.dumps(dt, 3)
+    self._test_pickle(dt, data)
+
+  def test_pickle_dumps_loads(self):
+    for i in (
+      '2007/01/02 12:34:56.789',
+      '2007/01/02 12:34:56.789 GMT+0200',
+      '2007/01/02 12:34:56.789 JST',
+      '2007/01/02 12:34:56.789 +0300',
+      '2007/01/02 12:34:56.789 +0430',
+      '2007/01/02 12:34:56.789 +1237',
+    ):
+      dt = DateTime(i)
+      self._test_pickle(dt, pickle.dumps(dt, 1))
+
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestDateUtils))
   suite.addTest(unittest.makeSuite(TestPinDateTime))
+  suite.addTest(unittest.makeSuite(TestTimeZoneContext))
+  suite.addTest(unittest.makeSuite(TestDateTimePatch))
+
+  # also run original tests from DateTime module
+  # pylint:disable=no-name-in-module
+  try:
+    import DateTime.tests.testDateTime as test_datetime
+  except ImportError:
+    from DateTime.tests import test_datetime
+  # pylint:enable=no-name-in-module
+
+  class DateTimeTests(test_datetime.DateTimeTests):
+    testTimezoneNaiveHandling = unittest.expectedFailure(
+      test_datetime.DateTimeTests.testTimezoneNaiveHandling)
+
+    if hasattr(test_datetime.DateTimeTests, 'test_intl_format_hyphen'):
+      test_intl_format_hyphen = unittest.expectedFailure(
+        test_datetime.DateTimeTests.test_intl_format_hyphen)
+
+  suite.addTest(unittest.makeSuite(DateTimeTests))
+
   return suite

bt5/erp5_web_renderjs_ui_test/ExtensionTemplateItem/portal_components/extension.erp5.WebRenderJSTestUtils.py

-import os, time
-from DateTime import DateTime
+from erp5.component.module.DateUtils import timeZoneContext
+
+current_timezone_contexts = []
 
 def setTimezone(timezone):
-  # timezone must be for example GMT-7
-  os.environ['TZ'] = timezone
-  time.tzset()
-  DateTime._isDST = False
-  DateTime._localzone = DateTime._localzone0 = DateTime._localzone1 = timezone
+  """Change the default timezone to `timezone`.
+  """
+  if current_timezone_contexts:
+    resetTimeZone()
+
+  tzc = timeZoneContext(timezone)
+  tzc.__enter__()
+  current_timezone_contexts.append(tzc)
+
   return "Timezone Updated"
+
+
+def resetTimeZone():
+  """Reset the timezone that might have been set by `setTimezone`
+  """
+  current_timezone_contexts.pop().__exit__(None, None, None)

product/ERP5/bootstrap/erp5_core/ModuleComponentTemplateItem/portal_components/module.erp5.DateUtils.py

@@ -27,11 +27,18 @@
 #
 ##############################################################################
 
+import contextlib
+import os
+from datetime import datetime
+from string import zfill
+import sys
+import time
 import warnings
 
+import mock
+import pytz
 from AccessControl import ModuleSecurityInfo
 from DateTime import DateTime
-from datetime import datetime
 import six
 
 security = ModuleSecurityInfo(__name__)
@@ -523,3 +530,35 @@ def copyDate(date, year=None, month=None, day=None,
   return DateTime('%i/%i/%i %i:%i:%d %s' % (year, month, day,
                                             hour, minute, second,
                                             timezone))
+
+
+@contextlib.contextmanager
+def timeZoneContext(timezone):
+  """Context manager to change timezone in tests.
+  """
+  saved_TZ = os.environ.get('TZ')
+  os.environ['TZ'] = timezone
+  time.tzset()
+  if timezone in pytz.all_timezones:
+    _multipleZones = time.daylight
+    _localzone0 = time.tzname[0]
+    _localzone1 = time.tzname[1] if time.daylight else time.tzname[0]
+  else:
+    _multipleZones = False
+    _localzone0 = _localzone1 = timezone
+  if hasattr(sys.modules['DateTime.DateTime'].DateTime, '_localzone0'):
+    patch_target = sys.modules['DateTime.DateTime'].DateTime
+  else:
+    # BBB DateTime 2
+    patch_target = sys.modules['DateTime.DateTime']
+
+  try:
+    with mock.patch.object(patch_target, '_localzone0', new=_localzone0), \
+        mock.patch.object(patch_target, '_localzone1', new=_localzone1), \
+        mock.patch.object(patch_target, '_multipleZones', new=_multipleZones):
+      yield
+  finally:
+    os.environ.pop('TZ')
+    if saved_TZ:
+      os.environ['TZ'] = saved_TZ
+    time.tzset()

product/ERP5OOo/tests/testOOoImport.py

@@ -241,7 +241,7 @@ class TestOOoImport(TestOOoImportMixin):
       sorted(['male' for i in range(num)]),
       sorted([person_list[i].getGender() for i in range(num)]))
     self.assertEqual(
-      sorted([DateTime('2008/02/%02d %s' % (i+1, 'GMT')) for i in range(num)]),
+      sorted([DateTime('2008-02-%02d' % (i+1)) for i in range(num)]),
       sorted([person_list[i].getStartDate() for i in range(num)]))
 
   def stepCheckImportFloatsAndPercentage(self, sequence=None, sequence_list=None, **kw):

product/ERP5Type/patches/DateTimePatch.py

@@ -35,9 +35,21 @@ SyntaxError, DateError, TimeError, localtime, time
 
 STATE_KEY = 'str'
 
-original_DateTime__eq__ = DateTimeKlass.__eq__
+# DateTime 3 changed the __eq__ behavior and d1 == d2 only if they have the same same
+# timezone. With DateTime 2 two dates from different timezones representing the same
+# time were equal. This patch keeps the behavior from DateTime 2.
+# See zopefoundation/DateTime commit fff6d04 (Various cleanups, improve unpickling
+# speed and distinguish between equal representations and references to equal points
+# in time., 2011-05-06)
 DateTimeKlass.__eq__ = DateTimeKlass.equalTo
 
+
+# ERP5 Patch for different pickle implementation, to optimize for disk usage.
+# We had different __getstate__ implementations, so we need __setstate__ to support
+# loading these formats that might be present in ZODBs.
+# This patch does not have support for timezone naive flag, because we don't need it
+# so far and also probably because we did not notice that it was added in original
+# DateTime.
 original_DateTime__setstate__ = DateTimeKlass.__setstate__
 
 def DateTime__setstate__(self, state):
@@ -45,6 +57,7 @@ def DateTime__setstate__(self, state):
     self.__dict__.clear()
   except AttributeError:
     pass
+  self._timezone_naive = False
   if isinstance(state, tuple):
     t, tz = state
     ms = (t - math.floor(t))
@@ -66,13 +79,13 @@ def DateTime__getstate__(self):
 
 DateTimeKlass.__getstate__ = DateTime__getstate__
 
-try:
-  DateTimeKlass._month_len
-except AttributeError:
-  # BBB
-  from DateTime.DateTime import _MONTH_LEN
-  DateTimeKlass._month_len = _MONTH_LEN
 
+# ERP5 Patch to have different parsing rules.
+# We have a patch since e0eba4791a (Authorised date manipulation before
+# year 1000, 2008-01-28), which replaced the method with an implementation
+# that did not change since, so we don't have new behaviors of DateTime
+# the most visible change might be that we don't have "timezone naive"
+# support.
 def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
   # Parse date-time components from a string
   month=year=tz=tm=None
@@ -107,6 +120,9 @@ def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
   else: tz = None  # Decide later, since the default time zone
   # could depend on the date.
 
+  # XXX we don't support timezone naive in this patch
+  self._timezone_naive = False
+
   ints,dels=[],[]
   i,l=0,len(st)
   while i < l:
@@ -273,18 +289,11 @@ def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
 
 DateTimeKlass._parse = DateTime_parse
 
-if __name__ == '__main__':
-  for i in ('2007/01/02 12:34:56.789',
-            '2007/01/02 12:34:56.789 GMT+0200',
-            '2007/01/02 12:34:56.789 JST',
-            '2007/01/02 12:34:56.789 +0300',
-            '2007/01/02 12:34:56.789 +0430',
-            '2007/01/02 12:34:56.789 +1237',
-            ):
-    a = DateTimeKlass(i)
-    b = DateTimeKlass()
-    b.__setstate__(a.__getstate__())
-    print(a, a.__dict__ == b.__dict__)
-    for i in a.__dict__.keys():
-      if a.__dict__[i] != b.__dict__[i]:
-        print(i, a.__dict__[i], b.__dict__[i])
+
+# DateTime 3 removed exceptions as class attributes (since
+# zopefoundation/DateTime commit 8114618 ), but we have some code expecting
+# these attributes, so undo this patch for convenience.
+DateTimeKlass.DateTimeError = DateTimeError
+DateTimeKlass.SyntaxError = SyntaxError
+DateTimeKlass.DateError = DateError
+DateTimeKlass.TimeError = TimeError

product/ERP5Type/tests/ERP5TypeTestCase.py

@@ -388,18 +388,11 @@ class ERP5TypeTestCaseMixin(ProcessingNodeTestCase, PortalTestCase, functional.F
       self.pinDateTime(None)
 
     def setTimeZoneToUTC(self):
-      # Make sure tests runs with UTC timezone. Some tests are checking values
-      # based on now, and this could give unexpected results:
-      # DateTime("2016/10/31") - DateTime("2016/10/30") = 1.0416666666666667 if
-      # you are running on a timezone like Europe/Paris, while it return 1.0 for
-      # UTC
-      os.environ['TZ'] = "UTC"
-      time.tzset()
-
-      mock.patch.object(sys.modules['DateTime.DateTime'], '_localzone0', new='UTC').start()
-      mock.patch.object(sys.modules['DateTime.DateTime'], '_localzone1', new='UTC').start()
-      mock.patch.object(sys.modules['DateTime.DateTime'], '_multipleZones', new=False).start()
-
+      # Deprecated, prefer using `timeZoneContext` context manager instead.
+      from erp5.component.module.DateUtils import timeZoneContext
+      timezone = timeZoneContext('UTC')
+      timezone.__enter__()
+      self.addCleanup(timezone.__exit__, None, None, None)
 
     def getDefaultSystemPreference(self):
       id = 'default_system_preference'

product/ZSQLCatalog/SearchKey/DateTimeKey.py

@@ -31,15 +31,17 @@ from __future__ import absolute_import
 
 import six
 from six import string_types as basestring
+import calendar
 from .SearchKey import SearchKey
 from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
 from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
 from zLOG import LOG
 from DateTime.DateTime import DateTime, DateTimeError
+from DateTime import Timezones
 try:
   from DateTime.DateTime import _TZINFO as _cache
 except ImportError:
-  # BBB version < zope4
+  # BBB DateTime 2
   from DateTime.DateTime import _cache
 from Products.ZSQLCatalog.interfaces.search_key import ISearchKey
 from zope.interface.verify import verifyClass
@@ -111,7 +113,7 @@ def castDate(value, change_timezone=True):
 delimiter_list = ' -/.:,+'
 
 def getMonthLen(datetime):
-  return datetime._month_len[datetime.isLeapYear()][datetime.month()]
+  return calendar.monthrange(datetime.year(), datetime.month())[1]
 
 def getYearLen(datetime):
   return 365 + datetime.isLeapYear()