Merge remote-tracking branch 'upstream/master' into zope4py2

component/groonga/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/groonga/groonga-12.0.4.tar.gz
-md5sum = a7c6416301e5b5899ef517193c510bd0
+url = https://packages.groonga.org/source/groonga/groonga-12.0.7.tar.gz
+md5sum = 5ef412a6941994e623cb50d76a8be261
 groonga-plugin-dir =  @@LOCATION@@/lib/groonga/plugins/
 # temporary patch to respect more tokens in natural language mode.
 patches =

component/gtk-2/buildout.cfg

@@ -14,6 +14,7 @@ extends =
   ../libtool/buildout.cfg
   ../m4/buildout.cfg
   ../perl/buildout.cfg
+  ../python-2.7/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../xorg/buildout.cfg
   ../icu/buildout.cfg
@@ -81,7 +82,7 @@ configure-options =
   --disable-static
   --disable-gtk-doc-html
 environment =
-  PATH=${glib:location}/bin:${freetype:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  PATH=${python2.7:location}/bin:${glib:location}/bin:${freetype:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${bzip2:location}/include -I${zlib:location}/include
   LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${harfbuzz:location}/lib -Wl,-rpath=${harfbuzz:location}/lib

component/mariadb/buildout.cfg

@@ -99,8 +99,8 @@ post-install =
 # as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/mroonga/mroonga-12.04.tar.gz
-md5sum = 1d154e6cb4540f7be5791b5fb376a5de
+url = https://packages.groonga.org/source/mroonga/mroonga-12.07.tar.gz
+md5sum = bee7506e08deda9a607d85ed03b8b62e
 pre-configure =
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source
@@ -117,9 +117,8 @@ configure-options =
   --disable-document
 post-install =
   cp -rs ${mariadb:location}/lib/plugin/* ${:plugin-dir}
-  cat @@LOCATION@@/share/mroonga/install.sql @@LOCATION@@/share/mroonga/update.sql > ${:install-sql}
 plugin-dir = @@LOCATION@@/lib/plugin
-install-sql = @@LOCATION@@/share/mroonga/install-and-update.sql
+install-sql = @@LOCATION@@/share/mroonga/install.sql
 make-targets = plugindir=${:plugin-dir} install
 patch-options = -p1
 patches =

component/postfix/buildout.cfg

@@ -14,8 +14,8 @@ extends =
 [postfix]
 recipe = slapos.recipe.cmmi
 shared = true
-url = ftp://ftp.porcupine.org/mirrors/postfix-release/official/postfix-3.5.8.tar.gz
-md5sum = c7c55ccc1db2a30d35c3867c21fe7109
+url = http://cdn.postfix.johnriley.me/mirrors/postfix-release/official/postfix-3.7.2.tar.gz
+md5sum = 92ff3863194004f7d8a0893421ba4248
 location = @@LOCATION@@
 patch-options = -p1
 patches =

component/slapos/obs.cfg

@@ -31,6 +31,7 @@ init +=
   assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
     "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)" %
     (buildout['directory'], buildout['destdir'] + buildout['rootdir']))
+part = python3.7
 
 [python3-common]
 configure-options +=
@@ -85,8 +86,8 @@ pre-configure =
   sed -i 's#/opt/slapos/parts/dbus/lib/libdbus-1.la#${dbus:location}/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
 environment +=
   LD_LIBRARY_PATH=${dbus:location}/lib
-  PYTHON_INCLUDES=-I${python:location}/include/python${python:version}
-  PYTHON_LIBS=-L${python:location}/lib -lpython${python:version} -lpthread -ldl -lutil -lm
+  PYTHON_INCLUDES=-I${python:location}/include/python${python:version}m
+  PYTHON_LIBS=-L${python:location}/lib -lpython${python:version}m -lpthread -ldl -lutil -lm
 post-install =
   sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
 

software/caddy-frontend/test/test.py

@@ -302,7 +302,8 @@ class TestDataMixin(object):
       os.path.dirname(os.path.realpath(__file__)), 'test_data', filename)
 
     try:
-      test_data = open(test_data_file).read().strip()
+      with open(test_data_file) as fh:
+        test_data = fh.read().strip()
     except IOError:
       test_data = ''
 
@@ -498,31 +499,32 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
   headers.setdefault('Via', 'http/1.1 clientvia')
 
   session = requests.Session()
-  if source_ip is not None:
-    new_source = source.SourceAddressAdapter(source_ip)
-    session.mount('http://', new_source)
-    session.mount('https://', new_source)
-  socket_getaddrinfo = socket.getaddrinfo
-  try:
-    add_custom_dns(domain, port, TEST_IP)
-    socket.getaddrinfo = new_getaddrinfo
-    # Use a prepared request, to disable path normalization.
-    # We need this because some test checks requests with paths like
-    # /test-path/deep/.././deeper but we don't want the client to send
-    # /test-path/deeper
-    # See also https://github.com/psf/requests/issues/5289
-    url = 'https://%s:%s/%s' % (domain, port, path)
-    req = requests.Request(
-        method='GET',
-        url=url,
-        headers=headers,
-        cookies=cookies,
-    )
-    prepped = req.prepare()
-    prepped.url = url
-    return session.send(prepped, verify=False, allow_redirects=False)
-  finally:
-    socket.getaddrinfo = socket_getaddrinfo
+  with session:
+    if source_ip is not None:
+      new_source = source.SourceAddressAdapter(source_ip)
+      session.mount('http://', new_source)
+      session.mount('https://', new_source)
+    socket_getaddrinfo = socket.getaddrinfo
+    try:
+      add_custom_dns(domain, port, TEST_IP)
+      socket.getaddrinfo = new_getaddrinfo
+      # Use a prepared request, to disable path normalization.
+      # We need this because some test checks requests with paths like
+      # /test-path/deep/.././deeper but we don't want the client to send
+      # /test-path/deeper
+      # See also https://github.com/psf/requests/issues/5289
+      url = 'https://%s:%s/%s' % (domain, port, path)
+      req = requests.Request(
+          method='GET',
+          url=url,
+          headers=headers,
+          cookies=cookies,
+      )
+      prepped = req.prepare()
+      prepped.url = url
+      return session.send(prepped, verify=False, allow_redirects=False)
+    finally:
+      socket.getaddrinfo = socket_getaddrinfo
 
 
 def fakeHTTPResult(domain, path, port=HTTP_PORT,
@@ -540,17 +542,18 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
   headers.setdefault('Via', 'http/1.1 clientvia')
   headers['Host'] = '%s:%s' % (domain, port)
   session = requests.Session()
-  if source_ip is not None:
-    new_source = source.SourceAddressAdapter(source_ip)
-    session.mount('http://', new_source)
-    session.mount('https://', new_source)
+  with session:
+    if source_ip is not None:
+      new_source = source.SourceAddressAdapter(source_ip)
+      session.mount('http://', new_source)
+      session.mount('https://', new_source)
 
-  # Use a prepared request, to disable path normalization.
-  url = 'http://%s:%s/%s' % (TEST_IP, port, path)
-  req = requests.Request(method='GET', url=url, headers=headers)
-  prepped = req.prepare()
-  prepped.url = url
-  return session.send(prepped, allow_redirects=False)
+    # Use a prepared request, to disable path normalization.
+    url = 'http://%s:%s/%s' % (TEST_IP, port, path)
+    req = requests.Request(method='GET', url=url, headers=headers)
+    prepped = req.prepare()
+    prepped.url = url
+    return session.send(prepped, allow_redirects=False)
 
 
 class TestHandler(BaseHTTPRequestHandler):
@@ -742,12 +745,15 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     server_process = multiprocessing.Process(
       target=server.serve_forever, name='HTTPServer')
     server_process.start()
+    # from now on, socket is used by server subprocess, we can close it
+    server.socket.close()
     cls.logger.debug('Started process %s' % (server_process,))
 
     cls.backend_https_url = 'https://%s:%s/' % server_https.server_address
     server_https_process = multiprocessing.Process(
       target=server_https.serve_forever, name='HTTPSServer')
     server_https_process.start()
+    server_https.socket.close()
     cls.logger.debug('Started process %s' % (server_https_process,))
 
     class NetlocHandler(TestHandler):
@@ -759,6 +765,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     netloc_a_http_process = multiprocessing.Process(
       target=netloc_a_http.serve_forever, name='netloc-a-http')
     netloc_a_http_process.start()
+    netloc_a_http.socket.close()
 
     netloc_b_http = ThreadedHTTPServer(
       (cls._ipv4_address, cls._server_netloc_b_http_port),
@@ -766,6 +773,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     netloc_b_http_process = multiprocessing.Process(
       target=netloc_b_http.serve_forever, name='netloc-b-http')
     netloc_b_http_process.start()
+    netloc_b_http.socket.close()
 
     cls.server_process_list = [
       server_process,
@@ -821,6 +829,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     self.server_https_auth_process = multiprocessing.Process(
       target=server_https_auth.serve_forever, name='HTTPSServerAuth')
     self.server_https_auth_process.start()
+    server_https_auth.socket.close()
     self.logger.debug('Started process %s' % (self.server_https_auth_process,))
 
   def stopAuthenticatedServerProcess(self):
@@ -840,8 +849,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     assert ca_certificate.status_code == http.client.OK
     cls.kedifa_caucase_ca_certificate_file = os.path.join(
       cls.working_directory, 'kedifa-caucase.ca.crt.pem')
-    open(cls.kedifa_caucase_ca_certificate_file, 'w').write(
-        ca_certificate.text)
+    with open(cls.kedifa_caucase_ca_certificate_file, 'w') as fh:
+      fh.write(ca_certificate.text)
 
   @classmethod
   def _fetchBackendClientCaCertificateFile(cls, parameter_dict):
@@ -850,8 +859,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     assert ca_certificate.status_code == http.client.OK
     cls.backend_client_caucase_ca_certificate_file = os.path.join(
       cls.working_directory, 'backend-client-caucase.ca.crt.pem')
-    open(cls.backend_client_caucase_ca_certificate_file, 'w').write(
-      ca_certificate.text)
+    with open(cls.backend_client_caucase_ca_certificate_file, 'w') as fh:
+      fh.write(ca_certificate.text)
 
   @classmethod
   def setUpMaster(cls):
@@ -1391,9 +1400,10 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
         self.instance_path, '*', 'var', 'log', 'httpd', log_name
       ))[0]
 
-    self.assertRegex(
-      open(log_file, 'r').readlines()[-1],
-      log_regexp)
+    with open(log_file) as fh:
+      self.assertRegex(
+        fh.readlines()[-1],
+        log_regexp)
 
 
 class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
@@ -2050,10 +2060,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     # check that monitor cors domains are correctly setup by file presence, as
     # we trust monitor stack being tested in proper place and it is too hard
     # to have working monitor with local proxy
-    self.assertTestData(
-      open(
-        os.path.join(
-          partition_path, 'etc', 'httpd-cors.cfg'), 'r').read().strip())
+    with open(os.path.join(
+          partition_path, 'etc', 'httpd-cors.cfg')) as fh:
+      self.assertTestData(fh.read().strip())
 
   def test_node_information_json(self):
     node_information_file_path = glob.glob(os.path.join(
@@ -2100,10 +2109,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
   def test_slave_partition_state(self):
     partition_path = self.getSlavePartitionPath()
-    self.assertTrue(
-      '-grace 2s' in
-      open(os.path.join(partition_path, 'bin', 'caddy-wrapper'), 'r').read()
-    )
+    with open(os.path.join(partition_path, 'bin', 'caddy-wrapper')) as fh:
+      self.assertIn('-grace 2s', fh.read())
 
   def test_monitor_conf(self):
     monitor_conf_list = glob.glob(
@@ -2112,8 +2119,11 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       ))
     self.assertEqual(3, len(monitor_conf_list))
     expected = [(False, q) for q in monitor_conf_list]
-    got = [('!py!' in open(q).read(), q) for q in monitor_conf_list]
-    # check that no monitor.conf in generated configuratio has magic !py!
+    got = []
+    for q in monitor_conf_list:
+      with open(q) as fh:
+        got.append(('!py!' in fh.read(), q))
+    # check that no monitor.conf in generated configuration has magic !py!
     self.assertEqual(
       expected,
       got
@@ -2173,14 +2183,15 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
   def test_server_polluted_keys_removed(self):
     buildout_file = os.path.join(
       self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
-    for line in [
-      q for q in open(buildout_file).readlines()
-      if q.startswith('config-slave-list') or q.startswith(
-          'config-extra_slave_instance_list')]:
-      self.assertFalse('slave_title' in line)
-      self.assertFalse('slap_software_type' in line)
-      self.assertFalse('connection-parameter-hash' in line)
-      self.assertFalse('timestamp' in line)
+    with open(buildout_file) as fh:
+      for line in [
+        q for q in fh.readlines()
+        if q.startswith('config-slave-list') or q.startswith(
+            'config-extra_slave_instance_list')]:
+        self.assertFalse('slave_title' in line)
+        self.assertFalse('slap_software_type' in line)
+        self.assertFalse('connection-parameter-hash' in line)
+        self.assertFalse('timestamp' in line)
 
   def assertBackendHeaders(
     self, backend_header_dict, domain, source_ip=SOURCE_IP, port=HTTPS_PORT,
@@ -3680,10 +3691,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       os.path.join(
         self.instance_path, '*', 'etc', 'caddy-slave-conf.d', '_ciphers.conf'
       ))[0]
-    self.assertTrue(
-      'ciphers RSA-3DES-EDE-CBC-SHA RSA-AES128-CBC-SHA'
-      in open(configuration_file).read()
-    )
+    with open(configuration_file) as fh:
+      self.assertIn(
+        'ciphers RSA-3DES-EDE-CBC-SHA RSA-AES128-CBC-SHA',
+        fh.read())
 
   def test_enable_cache_custom_domain(self):
     parameter_dict = self.assertSlaveBase(
@@ -5867,10 +5878,10 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
         self.instance_path, '*', 'etc', 'caddy-slave-conf.d',
         '_default_ciphers.conf'
       ))[0]
-    self.assertTrue(
-      'ciphers ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384'
-      in open(configuration_file).read()
-    )
+    with open(configuration_file) as fh:
+      self.assertIn(
+        'ciphers ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384',
+        fh.read())
 
   def test_own_ciphers(self):
     parameter_dict = self.assertSlaveBase('own_ciphers')
@@ -5893,10 +5904,10 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
         self.instance_path, '*', 'etc', 'caddy-slave-conf.d',
         '_own_ciphers.conf'
       ))[0]
-    self.assertTrue(
-      'ciphers ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256'
-      in open(configuration_file).read()
-    )
+    with open(configuration_file) as fh:
+      self.assertIn(
+        'ciphers ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256',
+        fh.read())
 
 
 class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):

software/slapos-sr-testing/software.cfg

@@ -401,7 +401,7 @@ PyPDF2 = 1.26.0+SlapOSPatched001
 Django = 1.11
 
 mock = 2.0.0:whl
-testfixtures = 6.11
+testfixtures = 6.11.0
 funcsigs = 1.0.2
 mysqlclient = 1.3.12
 pexpect = 4.8.0

stack/erp5/buildout.hash.cfg

@@ -50,15 +50,15 @@ md5sum = f928b9dc99f7f970caadfe7dd6f95d34
 
 [template-postfix]
 filename = instance-postfix.cfg.in
-md5sum = 0b7db23344b4795c8786d5e655810b38
+md5sum = 8f7bfca893a01c390df7a3dc9c2410e1
 
 [template-postfix-master-cf]
 filename = postfix_master.cf.in
-md5sum = 9ac81647368068a1a98a785d08074b43
+md5sum = ef164517e3f7170d03499967d625c3bb
 
 [template-postfix-main-cf]
 filename = postfix_main.cf.in
-md5sum = 886d1b004049c8dc1f89cbbfe4f95371
+md5sum = e9f03c66627beb4054d45123450162d2
 
 [template-postfix-aliases]
 filename = postfix_aliases.in
@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = 5747c487edd721d8a53061392307b55f
+md5sum = 2cf8721b82767c2461b5c847d281ae7b
 
 [template-erp5]
 filename = instance-erp5.cfg.in

stack/erp5/instance-postfix.cfg.in

@@ -36,6 +36,7 @@ bin = ${buildout:directory}/bin
 usr = ${buildout:directory}/usr
 srv = ${buildout:directory}/srv
 var = ${buildout:directory}/var
+var-log = ${:var}/log
 var-lib = ${:var}/lib
 var-lib-postfix = ${:var-lib}/postfix
 var-spool = ${:var}/spool
@@ -92,6 +93,7 @@ relayhost = {{ dumps(relay.get('host')) }}
 relay-sasl-credential = {{ dumps(relay.get('sasl-credential')) }}
 cyrus-sasldb = ${directory:etc-cyrus}/postfix.gdbm
 milter-list = {{ dumps(milter_list) }}
+xz-utils-location = {{ dumps(parameter_dict['xz-utils-location']) }}
 
 [userinfo]
 recipe = slapos.cookbook:userinfo
@@ -141,6 +143,16 @@ command =
 update =
   ${:openssl} req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout '${:key}' -out '${:cert}'
 
+[{{ section('postfix-logrotate') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = postfix-logrotate
+frequency = 0 0 * * *
+command = ${directory:bin}/postfix logrotate
+
+[postfix-main-cf-parameter]
+postfix-location = {{ parameter_dict['postfix-location'] }}
+
 [{{ section('postfix-main-cf') }}]
 < = jinja2-template-base
 output = ${directory:etc-postfix}/main.cf
@@ -163,6 +175,10 @@ context =
     key key smtpd-ssl:key
     key dh_512 smtpd-ssl:dh-512
     key dh_2048 smtpd-ssl:dh-2048
+    key log_directory directory:var-log
+    key xz_utils_location configuration:xz-utils-location
+    key postfix_location postfix-main-cf-parameter:postfix-location
+    key etc_postfix directory:etc-postfix
 
 [{{ section('postfix-master-cf') }}]
 < = jinja2-template-base

stack/erp5/instance.cfg.in

@@ -15,6 +15,7 @@ postfix-location = {{ postfix_location }}
 template-postfix-aliases = {{ template_postfix_aliases }}
 template-postfix-main-cf = {{ template_postfix_main_cf }}
 template-postfix-master-cf = {{ template_postfix_master_cf }}
+xz-utils-location =  {{ xz_utils_location }}
 
 [dynamic-template-postfix]
 < = jinja2-template-base

stack/erp5/postfix_main.cf.in

@@ -8,17 +8,19 @@ mail_owner = {{ mail_owner }}
 alias_maps = {{ aliases }}
 alias_database = {{ aliases }}
 mail_spool_directory = {{ spool_directory }}
-sendmail_path =
-newaliases_path =
-mailq_path =
+sendmail_path = {{ bin_directory }}/sendmail
+newaliases_path = {{ bin_directory }}/newaliases
+mailq_path = {{ bin_directory }}/mailq
 setgid_group = {{ setgid_group }}
-html_directory =
-manpage_directory =
-sample_directory =
-readme_directory =
+html_directory = no
+manpage_directory = {{ postfix_location }}/usr/local/man
+sample_directory = {{ postfix_location }}/etc/postfix
+readme_directory = no
 inet_interfaces = {{ inet_interfaces }}
 smtp_bind_address = 0.0.0.0
 smtp_bind_address6 = ::
+compatibility_level = 3.6
+smtputf8_enable = no
 
 # Compared to default:
 # - remove X-related variables, irrelevant for slapos, to be concise
@@ -69,3 +71,7 @@ smtp_sasl_auth_enable = yes
 smtp_sasl_password_maps = {{ sasl_passwd }}
 smtp_sasl_tls_security_options = noanonymous
 {%- endif %}
+
+maillog_file = {{ log_directory }}/postfix.log
+maillog_file_compressor = {{ xz_utils_location }}/bin/xz
+maillog_file_prefixes = {{ log_directory }}

stack/erp5/postfix_master.cf.in

@@ -27,3 +27,4 @@ virtual   unix  -       n       n       -       -       virtual
 lmtp      unix  -       -       n       -       -       lmtp
 anvil     unix  -       -       n       -       1       anvil
 scache    unix  -       -       n       -       1       scache
+postlog   unix-dgram n  -       n       -       1       postlogd

stack/slapos.cfg

@@ -194,7 +194,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.16.0
 slapos.cookbook = 1.0.253
-slapos.core = 1.7.13
+slapos.core = 1.8.0
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25
 slapos.rebootstrap = 4.5
@@ -263,7 +263,7 @@ Werkzeug = 1.0.1
 requests = 2.27.1
 charset-normalizer = 2.0.12
 pyparsing = 2.2.0
-certifi = 2020.04.05.1
+certifi = 2020.4.5.1
 
 [versions:sys.version_info < (3,8)]
 MarkupSafe = 1.0