Commit 397726e1 authored by Arnaud Fontaine's avatar Arnaud Fontaine

erp5/neoppod/slapos: Support for python3 from aurel/zope4 branch.

parent 3fb4c435
#!%(python_path)s
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
from __future__ import print_function
import socket
import sys
......@@ -9,9 +10,9 @@ expected = "%(expected-value)s"
not_expected = "%(expected-not-value)s"
if expected != "" and value != expected:
print "FAIL: %%s != %%s" %% (value, expected)
print("FAIL: %%s != %%s" %% (value, expected))
sys.exit(127)
if not_expected != "" and value == not_expected:
print "FAIL: %%s == %%s" %% (value, not_expected)
print("FAIL: %%s == %%s" %% (value, not_expected))
sys.exit(127)
......@@ -8,7 +8,7 @@ import pytz
def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
sleep = 0
with open(mysql_script_file) as script_file:
with open(mysql_script_file, 'rb') as script_file:
mysql_script = script_file.read()
mysql_list = mysql_binary, '-B'
mysql_tzinfo_to_sql_list = (
......
......@@ -57,7 +57,7 @@ class Recipe(GenericBaseRecipe):
# Prepare all filestorages
filestorage_snippet = ""
storage = self.options['storage']
if isinstance(storage, basestring):
if isinstance(storage, str):
for storage_definition in storage.splitlines():
storage_definition = storage_definition.strip()
if not storage_definition:
......
......@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed
[root-common]
filename = root-common.cfg.in
md5sum = c13b4f1a5aa526a8d3f8e02bf6baf785
md5sum = ae00507d9e69209a0babd725cf6be536
[instance-neo-admin]
filename = instance-neo-admin.cfg.in
......@@ -38,7 +38,7 @@ md5sum = a8939dcf440126a30db661b5a0a1bfb7
[template-neo]
filename = instance.cfg.in
md5sum = 9e63e16eda75e73ad4ffb50afde0505d
md5sum = 3c7770c03c51ee12d5a94f0924e98990
[cluster]
filename = cluster.cfg.in
......
......@@ -9,7 +9,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
template = {{ cluster }}
extra-context =
import re re
import urlparse urlparse
import urllib_parse six.moves.urllib.parse
import-list =
rawfile root_common {{ root_common }}
......
......@@ -2,13 +2,13 @@
{% set part_list = [] -%}
{% set sla_dict = {} -%}
{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
{% for sla, ref_list in six.iteritems(slapparameter_dict.get('sla-dict', {})) -%}
{% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
{% endfor -%}
{% macro sla(name, required=False) -%}
{% if required or name in sla_dict -%}
{% for k, (v,) in urlparse.parse_qs(sla_dict.pop(name), strict_parsing=1).iteritems() -%}
{% for k, (v,) in six.iteritems(urllib_parse.parse_qs(sla_dict.pop(name), strict_parsing=1)) -%}
sla-{{ k }} = {{ v }}
{% endfor -%}
{% else -%}
......@@ -133,7 +133,7 @@ config-monitor-port = {{ dumps(port) }}
{%- endif %}
config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
{%- endif %}
{%- for k, v in node.iteritems() %}
{%- for k, v in six.iteritems(node) %}
config-{{ k }} = {{ dumps(v) }}
{%- endfor %}
{{ sla(section_id) }}
......
......@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = f553aa7d6596dcf98e7e61bfb6bd81c7
md5sum = 4a1e279e3757c964caaa27137fecb66f
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -66,35 +66,35 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
[template-run-zelenium]
filename = run-zelenium-test.py.in
md5sum = 7a14019abf48ca100eb94d9add20f5ae
md5sum = 274365ebbade26558ca4836837e781aa
[template]
filename = instance.cfg.in
md5sum = 0444d646607f8643e987bb62d1cbbc1d
md5sum = ed280a76c22247cf2a05fac29b8a8b30
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = cb41e15a1585973896a7645dc2ae3a5c
md5sum = f5a1661449c9681b3de7d4af645124ba
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 1f33f3b93da32b34e2fd11471648835d
md5sum = 3c59315a8f102a970dc54ded85df735a
[template-zodb-base]
filename = instance-zodb-base.cfg.in
md5sum = bc821f9f9696953b10a03ad7b59a1936
md5sum = 0ac4b74436f554cd677f19275d18d880
[template-zope]
filename = instance-zope.cfg.in
md5sum = 5520e81f1fff3dc26568a614b595da37
md5sum = e24c7f5f6ba2beb31736cb3cf3d60768
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = cbcbe2daf3a51dfab50446a18c0be76e
md5sum = e0567c8d7717d1ed5daf4555566c76cd
[template-haproxy-cfg]
filename = haproxy.cfg.in
md5sum = 9d61e05c8578e0f17e349603ccaaf52c
md5sum = 4149e3e83c9981586ffa123d31a08908
[template-rsyslogd-cfg]
filename = rsyslogd.cfg.in
......
......@@ -150,7 +150,7 @@ defaults
{% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
{% set path_routing_list = parameter_dict['path-routing-list'] %}
{% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
{% for name, (port, _, certificate_authentication, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
listen family_{{ name }}
{%- if parameter_dict.get('ca-cert') -%}
{%- set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
......@@ -192,7 +192,7 @@ listen family_{{ name }}
{% endfor %}
{% for (ip, port), (_, backend_dict) in sorted(parameter_dict['zope-virtualhost-monster-backend-dict'].iteritems()) -%}
{% for (ip, port), (_, backend_dict) in sorted(six.iteritems(parameter_dict['zope-virtualhost-monster-backend-dict'])) -%}
{% set group_name = 'testrunner_' ~ loop.index0 %}
frontend frontend_{{ group_name }}
bind {{ ip }}:{{ port }} {{ bind_ssl_crt }}
......
......@@ -38,7 +38,7 @@ mode = 644
{% set frontend_caucase_url_hash_list = [] -%}
{% for frontend_caucase_url in frontend_caucase_url_list -%}
{% set hash = hashlib.md5(frontend_caucase_url).hexdigest() -%}
{% set hash = hashlib.md5(six.ensure_binary(frontend_caucase_url)).hexdigest() -%}
{% do frontend_caucase_url_hash_list.append(hash) -%}
{% set data_dir = '${directory:client-cert-ca}/%s' % hash -%}
{{ caucase.updater(
......@@ -120,9 +120,9 @@ update-command = ${:command}
{% set haproxy_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% set next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port'])) -%}
{% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%}
six.iteritems(slapparameter_dict['zope-family-dict'])) -%}
{% set zope_family_address_list = [] -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% set has_webdav = [] -%}
......@@ -306,7 +306,7 @@ certificate-and-key = ${directory:etc}/certificate-and-key-generated.pem
promise = check_socket_listening
name = haproxy.py
config-host = {{ ipv4 }}
config-port = {{ haproxy_dict.values()[0][0] }}
config-port = {{ next(six.itervalues(haproxy_dict))[0] }}
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
......
......@@ -28,7 +28,7 @@
{% set partition_thread_count_list = [] -%}
{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
{% set zope_family_override_dict = slapparameter_dict.get('family-override', {}) -%}
{% for zope_parameter_dict in zope_partition_dict.values() -%}
{% for zope_parameter_dict in six.itervalues(zope_partition_dict) -%}
{# Apply some zope_parameter_dict default values, to avoid duplication. -#}
{% do zope_parameter_dict.setdefault('thread-amount', 4) -%}
{% do zope_parameter_dict.setdefault('instance-count', 1) -%}
......@@ -49,7 +49,7 @@ config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-m
name = {{ name }}
software-type = {{ software_type }}
return = {{ ' '.join(ret) }}
{% for ret, publish in ret.iteritems() -%}
{% for ret, publish in six.iteritems(ret) -%}
{% if publish -%}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
{% endif -%}
......@@ -58,10 +58,10 @@ return = {{ ' '.join(ret) }}
{% endif -%}
{% endfor -%}
{{ root_common.sla(name) }}
{% for k, v in config.iteritems() -%}
{% for k, v in six.iteritems(config) -%}
config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
{% for k, v in key_config.iteritems() -%}
{% for k, v in six.iteritems(key_config) -%}
config-{{ k }} = {{ '${' ~ v ~ '}' }}
{% endfor -%}
config-name = {{ name }}
......@@ -177,7 +177,7 @@ connection-url = smtp://127.0.0.2:0/
{% endfor -%}
{% do assert(len(mountpoints) == len(zodb_dict)) -%}
{% set neo = [] -%}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% for server_type, server_dict in six.iteritems(storage_dict) -%}
{% if server_type == 'neo' -%}
{% set ((name, server_dict),) = server_dict.items() -%}
{% do neo.append(server_dict.get('cluster')) -%}
......@@ -196,7 +196,7 @@ connection-url = smtp://127.0.0.2:0/
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% for server_type, server_dict in six.iteritems(storage_dict) -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
......@@ -272,7 +272,7 @@ software-type = zope
{% set zope_backend_path_dict = {} -%}
{% set ssl_authentication_dict = {} -%}
{% set jupyter_zope_family_default = [] -%}
{% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
{% for custom_name, zope_parameter_dict in six.iteritems(zope_partition_dict) -%}
{% set partition_name = 'zope-' ~ custom_name -%}
{% set section_name = 'request-' ~ partition_name -%}
{% set promise_software_url_section_name = 'promise-software-url' ~ partition_name -%}
......@@ -461,7 +461,7 @@ recipe = slapos.cookbook:publish.serialised
Pick any published hosts-dict, they are expected to be identical - and there is
no way to check here.
-#}
hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
hosts-dict = {{ '${' ~ next(iter(zope_address_list_id_dict)) ~ ':connection-hosts-dict}' }}
{% endif -%}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
......
......@@ -195,16 +195,17 @@ environ =
{{ variable }}
{%- endfor %}
[odbc-ini-text]
text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
[{{ section('odbc-ini') }}]
< = jinja2-template-base
rendered = ${directory:etc}/odbc.ini
template = inline:{% raw -%}
{{ parameter_dict['text'].decode('base64') }}
{{ base64.b64decode(text).decode('utf-8') }}
{%- endraw %}
context = section parameter_dict odbc-ini-text
text = {{ dumps(base64.b64encode(slapparameter_dict.get('odbc-ini', '').encode('utf-8'))) }}
context =
key text :text
import base64 base64
[{{ section('logrotate-entry-mariadb') }}]
< = logrotate-entry-base
......
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% set ports = itertools.count(slapparameter_dict['tcpv4-port']) -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
{% set part_list = [] -%}
......@@ -19,13 +19,13 @@ ip = {{ ipv4 }}
{% set known_tid_storage_identifier_dict = {} -%}
{% set zodb_dict = {} -%}
{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
{% for name, zodb in six.iteritems(slapparameter_dict['zodb-dict']) -%}
{% do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
{% endfor -%}
{% set tidstorage_port = slapparameter_dict['tcpv4-port'] + len(zodb_dict) -%}
{% for family, zodb in zodb_dict.iteritems() -%}
{% for family, zodb in six.iteritems(zodb_dict) -%}
{% set storage_list = [] -%}
{% set current_port = next_port() -%}
{% set current_port = next(ports) -%}
{% set known_tid_storage_identifier_host = (ipv4, current_port), -%}
{% for name, zodb in zodb -%}
{% do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ current_port, 'storage': name}) %}
......@@ -140,7 +140,7 @@ input = inline: #!/bin/sh
zodb_backup_directory="{{ default_backup_path }}"
repozo="${tidstorage:repozo-binary}"
EXIT_CODE=0
{% for family, zodb in zodb_dict.iteritems() -%}
{% for family, zodb in six.iteritems(zodb_dict) -%}
{% for name, zodb in zodb -%}
{% set zeo_section_name = 'zeo-' ~ family %}
storage_name="{{ name }}"
......@@ -209,7 +209,7 @@ tidstorage = {{ tidstorage_repozo_path }}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-httpd-port = {{ next(ports) }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
......
......@@ -4,7 +4,7 @@
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% for name, zodb in six.iteritems(zodb_dict) -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'zeo' -%}
{% do storage_dict.update(zeo_dict.get(name, ())) -%}
......
......@@ -2,7 +2,7 @@
{% set wsgi = slapparameter_dict['wsgi'] -%}
{% set webdav = slapparameter_dict['webdav'] -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
{% set ports = itertools.count(slapparameter_dict['port-base']) -%}
{% set site_id = slapparameter_dict['site-id'] -%}
{% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
{% set node_id_base = slapparameter_dict['name'] -%}
......@@ -36,14 +36,14 @@ partition. No more (undefined result), no less (IndexError).
('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
('erp5-smtp', slapparameter_dict['smtp-url']),
) -%}
{% set parsed_url = urlparse.urlparse(url) -%}
{% set parsed_url = urllib_parse.urlparse(url) -%}
{% do port_dict.__setitem__(alias, parsed_url.port) -%}
{% do publishable_hosts_dict.__setitem__(alias, parsed_url.hostname) -%}
{%- endfor %}
{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
{% do publishable_hosts_dict.__setitem__(
'erp5-catalog-' ~ i,
urlparse.urlparse(url).hostname,
urllib_parse.urlparse(url).hostname,
) -%}
{%- endfor %}
{% do publishable_hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
......@@ -202,7 +202,7 @@ environment =
command-line = '${userhosts-bin:rendered}' ${:wrapped-command-line}
{# Hack to deploy SSL certs via instance parameters -#}
{% for zodb in zodb_dict.itervalues() -%}
{% for zodb in six.itervalues(zodb_dict) -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'neo' and storage_dict.get('ssl', 1) -%}
{% for k, v in (('_ca', 'ca.crt'),
......@@ -243,7 +243,7 @@ file-list = {{ parameter_dict['site-zcml'] }}
[{{ section('zope-inituser') }}]
< = jinja2-template-base
rendered = ${directory:instance}/inituser
template = inline:{{ slapparameter_dict['inituser-login'] }}:{SHA}{{ hashlib.sha1(slapparameter_dict['inituser-password']).digest().encode('base64').rstrip() }}
template = inline:{{ slapparameter_dict['inituser-login'] }}:{SHA}{{ base64.b64encode(hashlib.sha1(slapparameter_dict['inituser-password'].encode('utf-8')).digest()) }}
mode = 600
once = ${:rendered}_done
......@@ -253,7 +253,7 @@ site-id = {{ site_id }}
{% if site_id -%}
mysql-url = {{ slapparameter_dict['mysql-url-list'][0] }}
inituser = {{ slapparameter_dict['inituser-login'] }}
{% set mysql = urlparse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
{% set mysql = urllib_parse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
{% set mysql_db = mysql.path.split('/')[1] -%}
sql-connection-string = {{ '%s@erp5-catalog-0:%s %s %s' % (
mysql_db, mysql.port, mysql.username, mysql.password) }}
......@@ -264,7 +264,7 @@ home = ${buildout:directory}
# We only want to change the hostname to 'erp5-cloudooo' if we use the internal
# cloudooo. We plan to remove the ability to have an internal one, so this
# heuristic is enough.
{% set cloudooo = urlparse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
{% set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
cloudooo-url = {{ (cloudooo if cloudooo.port == None else
cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
......@@ -317,7 +317,7 @@ z2-log = ${directory:log}/{{ name }}-Z2.log
node-id = {{ dumps(node_id_base ~ (node_id_index_format % index)) }}
{% set log_list = [] -%}
{% set import_set = set() -%}
{% for db_name, zodb in zodb_dict.iteritems() -%}
{% for db_name, zodb in six.iteritems(zodb_dict) -%}
{% do zodb.setdefault('pool-size', thread_amount) -%}
{% if zodb['type'] == 'neo' -%}
{% do import_set.add('neo.client') -%}
......@@ -402,7 +402,7 @@ post = test ! -s {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} || {{ bin_direc
{% for i in instance_index_list -%}
{{ zope(
i,
next_port(),
next(ports),
slapparameter_dict['longrequest-logger-timeout'],
slapparameter_dict['longrequest-logger-interval'],
) }}
......@@ -420,7 +420,7 @@ wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
{% if test_runner_enabled and test_runner_node_count -%}
{% for _ in range(test_runner_node_count) %}
{% do test_runner_address_list.append((ipv4, next_port())) %}
{% do test_runner_address_list.append((ipv4, next(ports))) %}
{% endfor %}
{% if selenium_server_configuration_dict -%}
......@@ -454,7 +454,7 @@ wrapper-path = ${buildout:bin-directory}/runTestSuite
{% set connection_string_list = [] -%}
{% for url in slapparameter_dict['mysql-test-url-list'] -%}
{% set parsed_url = urlparse.urlparse(url) -%}
{% set parsed_url = urllib_parse.urlparse(url) -%}
{% do connection_string_list.append(
'%s@%s:%s %s %s' % (
parsed_url.path.lstrip('/'),
......@@ -563,7 +563,7 @@ software-release-url = ${slap-connection:software-release-url}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-httpd-port = {{ next(ports) }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
......
......@@ -50,7 +50,7 @@ extra-context =
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
key openssl_location :openssl-location
import re re
import urlparse urlparse
import urllib_parse six.moves.urllib.parse
import-list =
file root_common context:root-common
file caucase context:caucase-jinja2-library
......@@ -80,6 +80,7 @@ extra-context =
section parameter_dict dynamic-template-balancer-parameters
import itertools itertools
import hashlib hashlib
import functools functools
import-list =
file caucase context:caucase-jinja2-library
......@@ -125,7 +126,8 @@ extra-context =
key buildout_directory buildout:directory
key root_common context:root-common
section parameter_dict dynamic-template-zope-parameters
import urlparse urlparse
import base64 base64
import urllib_parse six.moves.urllib.parse
import hashlib hashlib
import itertools itertools
import json json
......@@ -176,6 +178,7 @@ template = {{ template_mariadb }}
filename = instance-mariadb.cfg
extra-context =
section parameter_dict dynamic-template-mariadb-parameters
import base64 base64
# Keep a section for backward compatibility for removed types
# Once the section is removed, ghost instances will keep failing until
......
......@@ -64,7 +64,7 @@ def main():
if response.code == 200:
static_dict = json.loads(response.read())
activity_list = []
for _, value in static_dict.iteritems():
for _, value in six.iteritems(static_dict):
activity_list += value['line_list']
if len(activity_list) == 0:
break
......@@ -181,7 +181,7 @@ def main():
try:
test_result = tool.createTestResult(revision = revision,
test_name_list = test_line_dict.keys(),
test_name_list = list(test_line_dict),
node_title = args.test_node_title,
test_title = test_suite_title,
project_title = args.project_title)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment