Merge remote-tracking branch 'origin/zope4py2' into zope4py3

component/egg-patch/Zope/0001-fix-OFS.Image.File.PUT-Issue-1015.patch

+From d3885eadb919abfcc86a82431deb9b9916127602 Mon Sep 17 00:00:00 2001
+From: dieter <dieter@handshake.de>
+Date: Wed, 2 Mar 2022 08:31:58 +0100
+Subject: [PATCH] fix `OFS.Image.File.PUT` -- Issue 1015
+
+work around `cgi` bug
+
+make `flake8` happy
+---
+ src/OFS/Image.py | 47 ++++++++++++++++++++++++++++++++++++++++++++---
+ 1 file changed, 44 insertions(+), 3 deletions(-)
+
+diff --git a/src/OFS/Image.py b/src/OFS/Image.py
+index c7b012a5e..fde006deb 100644
+--- a/src/OFS/Image.py
++++ b/src/OFS/Image.py
+@@ -16,6 +16,8 @@
+ import struct
+ from email.generator import _make_boundary
+ from io import BytesIO
++from io import TextIOBase
++from tempfile import TemporaryFile
+ from warnings import warn
+ 
+ from six import PY2
+@@ -568,6 +570,16 @@ class File(
+                 self, REQUEST, manage_tabs_message=msg)
+ 
+     def _get_content_type(self, file, body, id, content_type=None):
++        """return content type or ``None``.
++
++        *file* usually is a ``FileUpload`` (like) instance; if this
++        specifies a content type, it is used. If *file*
++        is not ``FileUpload`` like, it is ignored and the
++        content type is guessed from the other parameters.
++
++        *body* is either a ``bytes`` or a ``Pdata`` instance
++        and assumed to be the *file* data.
++        """
+         headers = getattr(file, 'headers', None)
+         if headers and 'content-type' in headers:
+             content_type = headers['content-type']
+@@ -579,6 +591,13 @@ class File(
+         return content_type
+ 
+     def _read_data(self, file):
++        """return the data and size of *file* as tuple *data*, *size*.
++
++        *file* can be a ``bytes``, ``Pdata``, ``FileUpload`` or
++        (binary) file like instance.
++
++        For large files, *data* is a ``Pdata``, otherwise a ``bytes`` instance.
++        """
+         import transaction
+ 
+         n = 1 << 16
+@@ -656,13 +675,35 @@ class File(
+         """Handle HTTP PUT requests"""
+         self.dav__init(REQUEST, RESPONSE)
+         self.dav__simpleifhandler(REQUEST, RESPONSE, refresh=1)
+-        type = REQUEST.get_header('content-type', None)
+ 
++        type = REQUEST.get_header('content-type', None)
+         file = REQUEST['BODYFILE']
+ 
++        # Work around ``cgi`` bug
++        # ``cgi`` can turn the request body into a text file using
++        # the default encoding. ``File``, however, insists to work
++        # with bytes and binary files and forbids text.
++        # Convert back.
++        tfile = None
++        if isinstance(file, TextIOBase):  # ``cgi`` bug
++            if hasattr(file, "buffer"):
++                file = file.buffer  # underlying binary buffer
++            else:
++                from ZPublisher.HTTPRequest import default_encoding
++                tfile = TemporaryFile("wb+")
++                bufsize = 1 << 16
++                while True:
++                    data = file.read(bufsize)
++                    if not data:
++                        break
++                    tfile.write(data.encode(default_encoding))
++                file.seek(0, 0)
++                tfile.seek(0, 0)
++                file = tfile
++
+         data, size = self._read_data(file)
+-        if isinstance(data, str):
+-            data = data.encode('UTF-8')
++        if tfile is not None:
++            tfile.close()
+         content_type = self._get_content_type(file, data, self.__name__,
+                                               type or self.content_type)
+         self.update_data(data, content_type, size)
+-- 
+2.35.1
+

stack/erp5/buildout.cfg

 [buildout]
-
 extends =
 # Exact version of Zope
-  ztk-versions.cfg
-  zope-versions.cfg
   buildout.hash.cfg
+# This has to be before Zope versions.cfg otherwise eggs such as ZConfig have
+# their versions overridden when slapos.cfg is extends by some component...
+  ../../stack/slapos.cfg
+  https://raw.githubusercontent.com/zopefoundation/Zope/4.5.3/versions.cfg
   ../../component/fonts/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/ghostscript/buildout.cfg
@@ -315,7 +316,7 @@ recipe = plone.recipe.command
 stop-on-error = true
 genbt5list = ${erp5:location}/product/ERP5/bin/genbt5list
 command =
-  echo '${local-bt5-repository:list}' |xargs ${buildout:executable} ${:genbt5list}
+  echo '${local-bt5-repository:list}' |xargs ${eggs:bin-directory}/python ${:genbt5list}
 update-command = ${:command}
 
 [erp5_repository_list]
@@ -586,6 +587,8 @@ eggs = ${neoppod:eggs}
   Record
 # StructuredText
   Zope
+# Python3
+  ${my2to3-dev:egg}
 
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
@@ -611,12 +614,14 @@ extra-paths =
 patch-binary = ${patch:location}/bin/patch
 PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8
 PyPDF2-patch-options = -p1
-Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic-4.7.patch#85b0090e216cead0fc86c5c274450d96
+Acquisition-patches =  ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic-4.7.patch#85b0090e216cead0fc86c5c274450d96
 Acquisition-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1
 # Zope 4 patches
-Zope-patches = ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-OFS-XMLExportImport.patch#12a9b9db76b3cd9035b6032d516143e0
+Zope-patches =
+  ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-OFS-XMLExportImport.patch#12a9b9db76b3cd9035b6032d516143e0
+  ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-fix-OFS.Image.File.PUT-Issue-1015.patch#c706e2f572ad8cd37ed033fb5f873cbe
 Zope-patch-options = -p1
 
 [eggs-all-scripts]
@@ -646,6 +651,18 @@ scripts =
 depends =
   ${slapos-toolbox-dependencies:eggs}
 
+[my2to3-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/bminusl/my2to3.git
+branch = master
+git-executable = ${git:location}/bin/git
+develop = true
+
+[my2to3-dev]
+recipe = zc.recipe.egg:develop
+egg = my2to3
+setup = ${my2to3-repository:location}
+
 [versions]
 # See ../../software/neoppod/software-common.cfg for versions common with NEO:
 #  neoppod, mysqlclient, slapos.recipe.template
@@ -658,14 +675,12 @@ pysvn = 1.9.15+SlapOSPatched001
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
 PyPDF2 = 1.26.0+SlapOSPatched001
-Zope = 4.5.3+SlapOSPatched001
+Zope = 4.5.3+SlapOSPatched002
 ## https://lab.nexedi.com/nexedi/slapos/merge_requests/648
 pylint = 2.6.0
 
 # use newer version than specified in ZTK
-PasteDeploy = 1.5.2
 argparse = 1.4.0
-zope.dottedname = 4.1.0
 
 # modified version that works fine for buildout installation
 SOAPpy = 0.12.0nxd001
@@ -706,7 +721,6 @@ WSGIUtils = 0.7
 astroid = 1.3.8
 erp5diff = 0.8.1.7
 five.formlib = 1.0.4
-five.localsitemanager = 2.0.5
 google-api-python-client = 1.6.1
 httplib2 = 0.10.3
 huBarcode = 1.0.0
@@ -748,8 +762,6 @@ zbarlight = 2.3
 cloudpickle = 0.5.3
 dask = 0.18.1
 toolz = 0.9.0
-zope.globalrequest = 1.5
-waitress = 1.4.4
 xlrd = 1.1.0
 
 # Re-add for as it is required to be there for uninstallation
@@ -778,20 +790,12 @@ jedi = 0.15.1
 parso = 0.5.1
 yapf = 0.28.0
 z3c.etestbrowser = 3.0.1
-zope.testbrowser = 5.5.1
-WSGIProxy2 = 0.4.6
-WebTest = 2.0.33
-beautifulsoup4 = 4.8.2
-WebOb = 1.8.5
-soupsieve = 1.9.5
 eggtestinfo = 0.3
 
 oic = 0.15.1
 Beaker = 1.11.0
 Mako = 1.1.4
 pyjwkest = 1.4.2
-alabaster = 0.7.12
-future = 0.18.2
 pycryptodomex = 3.10.1
 
 strict-rfc3339 = 0.7
@@ -801,8 +805,6 @@ jsonpointer = 2.2
 
 
 # WIP Zope 4 ⚠
-zope.interface = 5.2.0 
-ZConfig = 3.5.0
 Products.CMFCore = 2.4.0
 Products.StandardCacheManagers = 4.1.0
 Products.ZSQLMethods = 3.14

stack/erp5/buildout.hash.cfg

@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 9c7bf69e24bc8964f3854a0bd3eebf2c
+md5sum = 233fe9447d6fa48b4565f6e21df19546
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -223,19 +223,6 @@ pem = {{dumps(storage_dict.pop(k))}}
 {% endfor -%}
 {# endhack -#}
 
-[runzope-base]
-<= run-common
-instance-home = ${directory:instance}
-{% if wsgi -%}
-wrapped-command-line = '{{ bin_directory }}/runwsgi' {% if webdav %}-w{% endif %} {{ ipv4 }}:${:port} {% if timerserver_interval %}--timerserver-interval={{ timerserver_interval }}{% endif %} '${:configuration-file}'
-{% else -%}
-wrapped-command-line = '{{ bin_directory }}/runzope' -C '${:configuration-file}'
-{%- endif %}
-{%- set private_dev_shm = slapparameter_dict['private-dev-shm'] %}
-{%- if private_dev_shm %}
-private-tmpfs = {{ private_dev_shm }} /dev/shm
-{%- endif %}
-
 [{{ section('zcml') }}]
 recipe = slapos.cookbook:copyfilelist
 target-directory = ${directory:instance-etc}
@@ -346,6 +333,19 @@ context =
   import os os
   import re re
 
+[runzope-base]
+<= run-common
+instance-home = ${directory:instance}
+{% if wsgi -%}
+wrapped-command-line = '{{ bin_directory }}/runwsgi' --event-log-file={{ '${' ~ conf_parameter_name ~ ':event-log}' }} --access-log-file={{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {% if webdav %}-w{% endif %} {{ ipv4 }}:${:port} {% if timerserver_interval %}--timerserver-interval={{ timerserver_interval }}{% endif %} '${:configuration-file}'
+{% else -%}
+wrapped-command-line = '{{ bin_directory }}/runzope' -C '${:configuration-file}'
+{%- endif %}
+{%- set private_dev_shm = slapparameter_dict['private-dev-shm'] %}
+{%- if private_dev_shm %}
+private-tmpfs = {{ private_dev_shm }} /dev/shm
+{%- endif %}
+
 [{{ section(name) }}]
 < = runzope-base
 wrapper-path = ${directory:service-on-watch}/{{ name }}
@@ -403,7 +403,7 @@ config-maximum-delay = {{ slapparameter_dict["zope-longrequest-logger-maximum-de
 < = logrotate-entry-base
 name = {{ name }}
 log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ conf_parameter_name ~ ':longrequest-logger-file}' }} {{ ' '.join(log_list) }}
-post = test ! -s {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} || {{ bin_directory }}/slapos-kill --pidfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} -s USR2
+copytruncate = true
 {% endmacro -%}
 
 {% for i in instance_index_list -%}

stack/erp5/zope-versions.cfg

-[versions]
-#Zope = 4.5.3
-Zope2 = 4.0
-# AccessControl 5+ no longer supports Zope 4.
-AccessControl = 4.2
-Acquisition = 4.7
-AuthEncoding = 4.2
-BTrees = 4.7.2
-Chameleon = 3.8.1
-DateTime = 4.3
-# DocumentTemplate 4+ no longer supports Zope 4.
-DocumentTemplate = 3.4
-ExtensionClass = 4.5.0
-Missing = 4.1
-MultiMapping = 4.1
-Paste = 3.5.0
-PasteDeploy = 2.1.1
-Persistence = 3.0
-Products.BTreeFolder2 = 4.2
-# ZCatalog 6+ no longer supports Zope 4.
-Products.ZCatalog = 5.2
-Record = 3.5
-RestrictedPython = 5.1
-WSGIProxy2 = 0.4.6
-WebOb = 1.8.6
-WebTest = 2.0.35
-ZConfig = 3.5.0
-ZEO = 5.2.2
-ZODB = 5.6.0
-ZServer = 4.0.2
-five.globalrequest = 99.1
-five.localsitemanager = 3.2.2
-funcsigs = 1.0.2
-future = 0.18.2
-ipaddress = 1.0.23
-# mock 4.0 and up requires Python 3
-mock = 3.0.5
-pbr = 5.5.1
-persistent = 4.6.4
-pytz = 2020.4
-roman = 3.3
-shutilwhich = 1.1.0
-six = 1.15.0
-transaction = 3.0.0
-waitress = 1.4.4
-z3c.pt = 3.3.0
-zExceptions = 4.1
-zc.lockfile = 2.0
-zdaemon = 4.3
-zodbpickle = 2.0.0
-zope.annotation = 4.7.0
-zope.browser = 2.3
-zope.browsermenu = 4.4
-zope.browserpage = 4.4.0
-zope.browserresource = 4.4
-zope.cachedescriptors = 4.3.1
-zope.component = 4.6.2
-zope.componentvocabulary = 2.2.0
-zope.configuration = 4.4.0
-zope.container = 4.4.0
-zope.contentprovider = 4.2.1
-zope.contenttype = 4.5.0
-zope.datetime = 4.2.0
-zope.deferredimport = 4.3.1
-zope.deprecation = 4.4.0
-zope.dottedname = 4.3
-zope.event = 4.5.0
-zope.exceptions = 4.4
-zope.filerepresentation = 5.0.0
-zope.formlib = 4.7.1
-zope.globalrequest = 1.5
-zope.hookable = 5.0.1
-zope.i18n = 4.7.0
-zope.i18nmessageid = 5.0.1
-zope.interface = 5.2.0
-zope.lifecycleevent = 4.3.0
-zope.location = 4.2
-zope.pagetemplate = 4.5.0
-zope.processlifetime = 2.3.0
-zope.proxy = 4.3.5
-zope.ptresource = 4.2.0
-zope.publisher = 5.2.1
-zope.ramcache = 2.3
-zope.schema = 6.0.0
-zope.security = 5.1.1
-zope.sendmail = 5.1
-zope.sequencesort = 4.1.2
-zope.site = 4.4.0
-zope.size = 4.3
-zope.structuredtext = 4.3
-zope.tal = 4.4
-zope.tales = 5.1
-zope.testbrowser = 5.5.1
-# Version 4.8+ dropped support for Python 3.5
-zope.testing = 4.7
-zope.testrunner = 5.2
-zope.traversing = 4.4.1
-zope.viewlet = 4.2.1

stack/erp5/ztk-versions.cfg

-[buildout]
-extends =
-    zope-versions.cfg
-versions = versions
-
-[versions]
-# Version pins for development and optional dependencies.
-Babel = 2.8.1
-Jinja2 = 2.11.2
-MarkupSafe = 1.1.1
-# Pygments 2.6.0 and up require Python 3
-Pygments = 2.5.2
-# Version 2.0+ needs Python 3.x
-Sphinx = 1.8.5
-alabaster = 0.7.12
-appdirs = 1.4.4
-attrs = 20.3.0
-backports.functools-lru-cache = 1.6.1
-beautifulsoup4 = 4.9.3
-bleach = 3.2.1
-buildout.wheel = 0.2.0
-# Version 2020.4.5.2 and up claim no Python 2 support
-certifi = 2020.4.5.1
-cffi = 1.14.3
-chardet = 3.0.4
-cmarkgfm = 0.4.2
-collective.recipe.cmd = 0.11
-collective.recipe.sphinxbuilder = 1.1
-collective.recipe.template = 2.1
-colorama = 0.4.4
-# configparser 5 and up require Python 3
-configparser = 4.0.2
-contextlib2 = 0.6.0.post1
-coverage = 5.3
-distlib = 0.3.1
-docutils = 0.16
-filelock = 3.0.12
-idna = 2.10
-imagesize = 1.2.0
-# tox and pluggy require importlib-metadata <1
-importlib-metadata = 0.23
-lxml = 4.6.1
-manuel = 1.10.1
-# Version 6+ needs Python 3.x
-more-itertools = 5.0.0
-mr.developer = 2.0.1
-nose = 1.3.7
-packaging = 20.4
-pathlib2 = 2.3.5
-pip = 20.2.4
-pkginfo = 1.6.1
-plone.recipe.command = 1.1
-pluggy = 0.13.1
-py = 1.9.0
-pycparser = 2.20
-pyparsing = 2.4.7
-python-gettext = 4.0
-readme-renderer = 28.0
-repoze.sphinx.autointerface = 0.8
-requests = 2.25.0
-requests-toolbelt = 0.9.1
-scandir = 1.10.0
-snowballstemmer = 2.0.0
-# soupsieve 2 needs Python 3
-soupsieve = 1.9.6
-sphinx-rtd-theme = 0.5.0
-sphinxcontrib-serializinghtml = 1.1.4
-sphinxcontrib-websupport = 1.2.4
-# tempstorage is only needed because the Sphinx documentation parses
-# ZConfig xml configurations, which still contain references to it
-tempstorage = 5.1
-toml = 0.10.2
-tox = 3.20.1
-tqdm = 4.51.0
-# Version 2+ needs Python 3.x
-twine = 1.15.0
-typing = 3.7.4.3
-urllib3 = 1.26.1
-virtualenv = 20.1.0
-webencodings = 0.5.1
-wheel = 0.35.1
-z3c.checkversions = 1.2
-zc.recipe.egg = 2.0.7
-zc.recipe.testrunner = 2.1
-zest.releaser = 6.22.1
-# Version 2 requires Python 3
-zipp = 1.1.1
-zodbupdate = 1.5

stack/logrotate/buildout.hash.cfg

@@ -18,8 +18,8 @@ md5sum = 7c41026716d856bba7c1252b72adbf77
 
 [logrotate-entry-template]
 filename = logrotate_entry.in
-md5sum = ce6ccdd52148770149e6e7525ab71e80
+md5sum = 02c1009f8e0dc371cfc1290afef72ec7
 
 [template-logrotate-base]
 filename = instance-logrotate-base.cfg.in
-md5sum = 8a774b677623c77c6ff0b88852fac643
+md5sum = 36a1dcb098d0d643cdaf988a58751aa6

stack/logrotate/instance-logrotate-base.cfg.in

@@ -53,12 +53,14 @@ rendered = ${logrotate-conf-parameter:logrotate-entries}/${:name}
 context =
   key backup :backup
   key log :log
+  key copytruncate :copytruncate
   key post :post
   key pre :pre
   key frequency :frequency
   key rotate_num :rotate-num
   key nocompress :nocompress
   key delaycompress :delaycompress
+copytruncate = false
 post =
 pre =
 frequency = daily

stack/logrotate/logrotate_entry.in

@@ -10,6 +10,7 @@
   create
   olddir {{ backup }}
   missingok
+  {% if copytruncate %}copytruncate{% endif %}
   {% if pre %}prerotate
     {{ pre }}
   endscript{% endif %}