Commit 78936adc authored by Jérome Perrin's avatar Jérome Perrin

Merge remote-tracking branch 'origin/master' into zope4py2

parents 6e378eef c2bfe70f
...@@ -48,11 +48,20 @@ md5sum = dbf727a4b0e365bf88d97cbfde590016 ...@@ -48,11 +48,20 @@ md5sum = dbf727a4b0e365bf88d97cbfde590016
environment-extra = environment-extra =
# build Go without testing it # build Go without testing it
# NOTE go1.4 does not have build cache # NOTE go1.4 does not have build cache
make-targets= cd src && unset GOBIN && ./make.bash && cp -alf .. ${:location} setarch =
make-targets= cd src && unset GOBIN && ${:setarch} ./make.bash && cp -alf .. ${:location}
# go1.4 is used for bootstrap and does not support CGO
environment-extra =
CGO_ENABLED=0
# skip-chown-tests.patch does not apply to go1.4, but we don't run go1.4 tests. # skip-chown-tests.patch does not apply to go1.4, but we don't run go1.4 tests.
patches = patches =
# go1.4 does not have support for arm64 - build it in arm32 mode
[golang14:platform.machine() == 'aarch64']
setarch = setarch arm
[golang1.12] [golang1.12]
<= golang-common <= golang-common
......
...@@ -75,7 +75,6 @@ scikit-learn = 0.18.1 ...@@ -75,7 +75,6 @@ scikit-learn = 0.18.1
statsmodels = 0.8.0 statsmodels = 0.8.0
terminado = 0.6 terminado = 0.6
tornado = 4.4.2 tornado = 4.4.2
traitlets = 4.3.3
# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source. # nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
nbconvert = 4.1.0 nbconvert = 4.1.0
......
[buildout] [buildout]
extends =
../patch/buildout.cfg
parts = parts =
libiconv libiconv
[libiconv] [libiconv]
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patches =
${:_profile_base_location_}/libiconv.gets.patch#8a20d8afe0617fce56f77537d2b84621
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.14.tar.gz url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.17.tar.gz
md5sum = e34509b1623cec449dfeb73d7ce9c6c6 md5sum = d718cd5a59438be666d1575855be72c3
--- libiconv-1.14.orig/srclib/stdio.in.h 2011-08-07 13:42:06.000000000 +0000
+++ libiconv-1.14/srclib/stdio.in.h 2013-01-09 19:56:21.115819812 +0000
@@ -680,22 +680,7 @@
#endif
#if @GNULIB_GETS@
-# if @REPLACE_STDIO_READ_FUNCS@ && @GNULIB_STDIO_H_NONBLOCKING@
-# if !(defined __cplusplus && defined GNULIB_NAMESPACE)
-# undef gets
-# define gets rpl_gets
-# endif
-_GL_FUNCDECL_RPL (gets, char *, (char *s) _GL_ARG_NONNULL ((1)));
-_GL_CXXALIAS_RPL (gets, char *, (char *s));
-# else
-_GL_CXXALIAS_SYS (gets, char *, (char *s));
-# undef gets
-# endif
-_GL_CXXALIASWARN (gets);
-/* It is very rare that the developer ever has full control of stdin,
- so any use of gets warrants an unconditional warning. Assume it is
- always declared, since it is required by C89. */
-_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
+#undef gets
#endif
...@@ -38,6 +38,7 @@ environment = ...@@ -38,6 +38,7 @@ environment =
PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:${python3:location}/bin:%(PATH)s PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:${python3:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${libsecret:pkg-config-path} PKG_CONFIG_PATH=${libsecret:pkg-config-path}
LDFLAGS=-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib LDFLAGS=-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib
NODE_OPTIONS=--max_old_space_size=4096
pre-configure = pre-configure =
mkdir -p $TMPDIR mkdir -p $TMPDIR
echo '${package.json:content}' > %(location)s/package.json echo '${package.json:content}' > %(location)s/package.json
......
...@@ -145,7 +145,7 @@ ...@@ -145,7 +145,7 @@
"type": "object" "type": "object"
}, },
"hosts-dict": { "hosts-dict": {
"description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-cloudooo, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)", "description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
"patternProperties": { "patternProperties": {
".*": { ".*": {
"description": "An IP or domain name to which current entry will resolve", "description": "An IP or domain name to which current entry will resolve",
...@@ -294,11 +294,14 @@ ...@@ -294,11 +294,14 @@
], ],
"type": "object" "type": "object"
}, },
"cloudooo-url": { "cloudooo-url-list": {
"description": "Format conversion service URL", "description": "Format conversion service URLs",
"pattern": "^https?://", "type": "array",
"type": "string", "items": {
"format": "uri" "pattern": "^https?://",
"type": "string",
"format": "uri"
}
}, },
"cloudooo-retry-count": { "cloudooo-retry-count": {
"description": "Define retry count for cloudooo in network error case in test", "description": "Define retry count for cloudooo in network error case in test",
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
import contextlib import contextlib
import glob import glob
import http.client
import json import json
import os import os
import shutil import shutil
...@@ -442,17 +443,20 @@ class ZopeSkinsMixin: ...@@ -442,17 +443,20 @@ class ZopeSkinsMixin:
def _setUpClass(cls): def _setUpClass(cls):
super()._setUpClass() super()._setUpClass()
param_dict = cls.getRootPartitionConnectionParameterDict() param_dict = cls.getRootPartitionConnectionParameterDict()
with cls.getXMLRPCClient() as erp5_xmlrpc_client: # wait for ERP5 to be ready and have processed all activities
# wait for ERP5 to be ready (TODO: this should probably be a promise) # from initial setup
for _ in range(120): for _ in range(120):
time.sleep(1) with cls.getXMLRPCClient() as erp5_xmlrpc_client:
try: try:
erp5_xmlrpc_client.getTitle() if erp5_xmlrpc_client.portal_activities.countMessage() == 0:
break
except (xmlrpc.client.ProtocolError, except (xmlrpc.client.ProtocolError,
xmlrpc.client.Fault): xmlrpc.client.Fault,
http.client.HTTPException):
pass pass
else: time.sleep(5)
break else:
raise AssertionError("ERP5 is not ready")
@classmethod @classmethod
def _getAuthenticatedZopeUrl(cls, path, family_name='default'): def _getAuthenticatedZopeUrl(cls, path, family_name='default'):
...@@ -902,3 +906,65 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase): ...@@ -902,3 +906,65 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'), self._getAuthenticatedZopeUrl('ERP5Site_doSlowRequest', family_name='no-timeout'),
verify=False, verify=False,
timeout=6) timeout=6)
class TestCloudooo(ZopeSkinsMixin, ERP5InstanceTestCase):
"""Test ERP5 can be instantiated with cloudooo parameters
"""
__partition_reference__ = 'c'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
'cloudooo-url-list': [
'https://cloudooo1.example.com/',
'https://cloudooo2.example.com/',
],
'cloudooo-retry-count': 123,
})
}
def test_cloudooo_url_list_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerUrlList'),
verify=False).text,
"['https://cloudooo1.example.com/', 'https://cloudooo2.example.com/']")
@unittest.expectedFailure # setting "retry" is not implemented
def test_cloudooo_retry_count_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerRetry'),
verify=False).text,
"123")
class TestCloudoooDefaultParameter(ZopeSkinsMixin, ERP5InstanceTestCase):
"""Test default ERP5 cloudooo parameters
"""
__partition_reference__ = 'cd'
def test_cloudooo_url_list_preference(self):
self.assertIn(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerUrlList'),
verify=False).text,
[
"['https://cloudooo1.erp5.net/', 'https://cloudooo.erp5.net/']",
"['https://cloudooo.erp5.net/', 'https://cloudooo1.erp5.net/']",
])
@unittest.expectedFailure # default value of "retry" does not match schema
def test_cloudooo_retry_count_preference(self):
self.assertEqual(
requests.get(
self._getAuthenticatedZopeUrl(
'portal_preferences/getPreferredDocumentConversionServerRetry'),
verify=False).text,
"2")
...@@ -62,7 +62,6 @@ scripts = ...@@ -62,7 +62,6 @@ scripts =
surykatka = 0.8.0 surykatka = 0.8.0
# For surykatka 0.7.1 # For surykatka 0.7.1
click = 8.0.1
dnspython = 2.1.0 dnspython = 2.1.0
forcediphttpsadapter = 1.0.1 forcediphttpsadapter = 1.0.1
miniupnpc = 2.0.2 miniupnpc = 2.0.2
......
...@@ -44,12 +44,3 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_} ...@@ -44,12 +44,3 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-zones-file] [template-zones-file]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_} url = ${:_profile_base_location_}/${:_update_hash_filename_}
[versions]
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
passlib = 1.7.1
GitPython = 2.1.11
feedparser = 5.2.1
atomize = 0.2.0
...@@ -220,10 +220,7 @@ validators = 0.12.2 ...@@ -220,10 +220,7 @@ validators = 0.12.2
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
ecdsa = 0.13 ecdsa = 0.13
gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
smmap = 0.9.0
furl = 2.1.0 furl = 2.1.0
orderedmultidict = 1.0.1 orderedmultidict = 1.0.1
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 6d9b94b50ea46b0b98c9019d74780876 md5sum = 2491969d49f8bf9b172e89b1c0e9d98e
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -21,6 +21,10 @@ ...@@ -21,6 +21,10 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %} {% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %} {% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%} {% endif -%}
{# Backward compatibility for cloudooo-url #}
{% if slapparameter_dict.get('cloudooo-url') -%}
{% do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%} {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%} {% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
...@@ -231,7 +235,7 @@ return = ...@@ -231,7 +235,7 @@ return =
{% endif -%} {% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }} config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
config-caucase-url = {{ dumps(caucase_url) }} config-caucase-url = {{ dumps(caucase_url) }}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
......
...@@ -146,8 +146,6 @@ scripts = ...@@ -146,8 +146,6 @@ scripts =
[versions] [versions]
Flask-Auth = 0.85 Flask-Auth = 0.85
futures = 3.0.5 futures = 3.0.5
gitdb = 0.6.4
gunicorn = 19.10.0 gunicorn = 19.10.0
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
smmap = 0.9.0
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance-theia] [instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in _update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = bd79a9e6306b321414b9f83524308e5f md5sum = 937f8ebdfa8112aafe11235a23fb85a9
[instance] [instance]
_update_hash_filename_ = instance.cfg.in _update_hash_filename_ = instance.cfg.in
......
...@@ -13,7 +13,9 @@ theia-environment-parts = ...@@ -13,7 +13,9 @@ theia-environment-parts =
settings.json settings.json
theia-parts = theia-parts =
frontend-reload frontend-instance
frontend-instance-rsyslogd
python-server
promises promises
parts = parts =
...@@ -90,11 +92,13 @@ recipe = ...@@ -90,11 +92,13 @@ recipe =
instance-promises = instance-promises =
$${theia-listen-promise:name} $${theia-listen-promise:name}
$${frontend-listen-promise:name} $${frontend-listen-promise:name}
$${python-server-listen-promise:name}
$${frontend-authentication-promise:name} $${frontend-authentication-promise:name}
$${remote-frontend-url-available-promise:name} $${remote-frontend-url-available-promise:name}
{% if additional_frontend %} {% if additional_frontend %}
$${remote-additional-frontend-url-available-promise:name} $${remote-additional-frontend-url-available-promise:name}
{% endif %} {% endif %}
$${frontend-instance-rsyslogd-promise:name}
$${slapos-standalone-listen-promise:name} $${slapos-standalone-listen-promise:name}
$${slapos-standalone-ready-promise:name} $${slapos-standalone-ready-promise:name}
$${slapos-autorun-promise:name} $${slapos-autorun-promise:name}
...@@ -116,6 +120,13 @@ name = $${:_buildout_section_name_}.py ...@@ -116,6 +120,13 @@ name = $${:_buildout_section_name_}.py
config-host = $${frontend-instance:ip} config-host = $${frontend-instance:ip}
config-port = $${frontend-instance:port} config-port = $${frontend-instance:port}
[python-server-listen-promise]
<= monitor-promise-base
promise = check_socket_listening
name = $${:_buildout_section_name_}.py
config-host = $${python-server-port:ip}
config-port = $${python-server-port:port}
[frontend-authentication-promise] [frontend-authentication-promise]
<= monitor-promise-base <= monitor-promise-base
promise = check_url_available promise = check_url_available
...@@ -142,6 +153,12 @@ config-url = $${remote-additional-frontend:connection-secure_access} ...@@ -142,6 +153,12 @@ config-url = $${remote-additional-frontend:connection-secure_access}
config-http-code = 401 config-http-code = 401
{% endif %} {% endif %}
[frontend-instance-rsyslogd-promise]
<= monitor-promise-base
promise = check_command_execute
name = rsyslogd_listen_promise.py
config-command = test -S $${frontend-instance-rsyslogd-config:log-socket}
[slapos-standalone-listen-promise] [slapos-standalone-listen-promise]
<= monitor-promise-base <= monitor-promise-base
promise = check_socket_listening promise = check_socket_listening
...@@ -207,7 +224,7 @@ sla-instance_guid = {{ parameter_dict['additional-frontend-guid'] }} ...@@ -207,7 +224,7 @@ sla-instance_guid = {{ parameter_dict['additional-frontend-guid'] }}
{% endif %} {% endif %}
# Local Caddy Frontend # Local Haproxy Frontend
# -------------------- # --------------------
[frontend-instance-password] [frontend-instance-password]
...@@ -215,6 +232,39 @@ recipe = slapos.cookbook:generate.password ...@@ -215,6 +232,39 @@ recipe = slapos.cookbook:generate.password
username = admin username = admin
storage-path = $${buildout:parts-directory}/.$${:_buildout_section_name_} storage-path = $${buildout:parts-directory}/.$${:_buildout_section_name_}
[frontend-instance-rsyslogd-config]
recipe = slapos.recipe.template
output = $${directory:etc}/$${:_buildout_section_name_}
log-file = $${directory:log}/frontend-instance.log
log-socket = $${directory:run}/rsyslog.sock
pidfile = $${directory:pidfiles}/rsyslogd.pid
inline =
module(
load="imuxsock"
SysSock.Name="$${:log-socket}")
# Just simply output the raw line without any additional information, as
# haproxy emits enough information by itself
# Also cut out first empty space in msg, which is related to rsyslogd
# internal and end up cutting on 8k, as it's default of $MaxMessageSize
template(name="rawoutput" type="string" string="%msg:2:8192%\n")
$ActionFileDefaultTemplate rawoutput
$FileCreateMode 0600
$DirCreateMode 0700
$Umask 0022
$WorkDirectory $${directory:run}
*.* $${:log-file};rawoutput
[frontend-instance-rsyslogd]
recipe = slapos.cookbook:wrapper
command-line = ${rsyslogd:location}/sbin/rsyslogd -i $${frontend-instance-rsyslogd-config:pidfile} -n -f $${frontend-instance-rsyslogd-config:output}
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
hash-files = $${frontend-instance-rsyslogd-config:output}
[frontend-instance-port] [frontend-instance-port]
recipe = slapos.cookbook:free_port recipe = slapos.cookbook:free_port
minimum = 3000 minimum = 3000
...@@ -224,61 +274,83 @@ ip = {{ ipv6_random }} ...@@ -224,61 +274,83 @@ ip = {{ ipv6_random }}
[frontend-instance-certificate] [frontend-instance-certificate]
recipe = plone.recipe.command recipe = plone.recipe.command
command = command =
if [ ! -e $${:key-file} ] if [ ! -e $${:cert-file} ]
then then
${openssl-output:openssl} req -x509 -nodes -days 3650 \ ${openssl-output:openssl} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \ -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-newkey rsa:1024 -keyout $${:key-file} \ -newkey rsa:1024 -keyout $${:cert-file} \
-out $${:cert-file} -out $${:cert-file}
fi fi
update-command = $${:command} update-command = $${:command}
key-file = $${directory:etc}/$${:_buildout_section_name_}.key cert-file = $${directory:etc}/$${:_buildout_section_name_}.pem
cert-file = $${directory:etc}/$${:_buildout_section_name_}.crt
common-name = $${frontend-instance-config:ip} common-name = $${frontend-instance-config:ip}
location = location =
$${:key-file}
$${:cert-file} $${:cert-file}
[frontend-instance-config] [frontend-instance-config]
recipe = slapos.recipe.template recipe = slapos.recipe.template
output = $${directory:etc}/$${:_buildout_section_name_} output = $${directory:etc}/$${:_buildout_section_name_}
blankline =
inline = inline =
:$${:port} { global
bind $${:ip} maxconn 4096
tls $${frontend-instance-certificate:cert-file} $${frontend-instance-certificate:key-file} master-worker
log stdout pidfile $${frontend-instance:pidfile}
errors stderr log $${frontend-instance-rsyslogd-config:log-socket} local0 info
gzip
# because caddy does not support upgrade http2 to websocket defaults
# https://tools.ietf.org/html/rfc8441 log global
tls { option httplog
alpn http/1.1 mode http
} retries 1
root $${directory:frontend-static} option redispatch
browse maxconn 2000
proxy / $${theia-instance:base-url} { balance roundrobin
except $${frontend-instance-fonts:folder-name} $${frontend-instance-slapos.css:folder-name} public $${favicon.ico:filename} $${frontend-instance-logo:filename} timeout connect 10s
} timeout queue 60s
proxy /services $${theia-instance:base-url} { timeout server 305s
websocket timeout client 305s
}
proxy /socket.io $${theia-instance:base-url} { # compress some content types
websocket compression algo gzip
} compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
basicauth $${frontend-instance-password:username} $${frontend-instance-password:passwd} {
realm "Theia"
/ userlist basic-auth-list
} user $${frontend-instance-password:username} insecure-password $${frontend-instance-password:passwd}
}
frontend app
log global
bind $${:ip}:$${:port} ssl crt $${frontend-instance-certificate:cert-file} alpn h2,http/1.1
# writing twice the same ACL is doing OR
acl is_public path_beg /public/
acl is_public path /$${favicon.ico:filename}
acl auth_ok http_auth(basic-auth-list)
# No authentication for public folder
http-request auth unless auth_ok || is_public
use_backend static if { path_beg /$${frontend-instance-fonts:folder-name} } || { path_beg /$${frontend-instance-slapos.css:folder-name} } || { path /$${frontend-instance-logo:filename} } || is_public
default_backend nodejs
backend nodejs
log global
server nodejs_backend $${theia-instance:ip}:$${theia-instance:port}
backend static
log global
server static_backend $${python-server-port:ip}:$${python-server-port:port}
$${:blankline}
ip = $${frontend-instance-port:ip} ip = $${frontend-instance-port:ip}
hostname = [$${:ip}] hostname = [$${:ip}]
port = $${frontend-instance-port:port} port = $${frontend-instance-port:port}
pidfile = $${directory:pidfiles}/haproxy.pid
[frontend-instance] [frontend-instance]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_} wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = command-line =
${caddy:output} -conf $${frontend-instance-config:output} -pidfile $${:pidfile} ${haproxy:location}/sbin/haproxy -f $${frontend-instance-config:output}
hash-files = $${frontend-instance-config:output}
ip = $${frontend-instance-config:ip} ip = $${frontend-instance-config:ip}
hostname = $${frontend-instance-config:hostname} hostname = $${frontend-instance-config:hostname}
...@@ -287,7 +359,7 @@ pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid ...@@ -287,7 +359,7 @@ pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
url = https://$${:hostname}:$${:port}/ url = https://$${:hostname}:$${:port}/
[frontend-instance-fonts] [frontend-instance-fonts]
; XXX caddy 1 does not seem to serve different folders at different locations ; XXX python server only serves one folder
; so we link fonts in static folder ; so we link fonts in static folder
recipe = plone.recipe.command recipe = plone.recipe.command
location = $${directory:frontend-static}/$${:folder-name} location = $${directory:frontend-static}/$${:folder-name}
...@@ -314,18 +386,6 @@ folder-name = css ...@@ -314,18 +386,6 @@ folder-name = css
context = context =
key logo_image frontend-instance-logo:filename key logo_image frontend-instance-logo:filename
[frontend-reload]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${bash:location}/bin/bash -c
"kill -s USR1 $$(${coreutils:location}/bin/cat $${frontend-instance:pidfile}) \
&& ${coreutils:location}/bin/sleep infinity"
hash-files =
$${frontend-instance-config:output}
$${frontend-instance:wrapper-path}
wait-for-files = $${frontend-instance:pidfile}
[favicon.ico] [favicon.ico]
# generate a pseudo random favicon, different for each instance name. # generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build recipe = slapos.recipe.build
...@@ -350,6 +410,20 @@ install = ...@@ -350,6 +410,20 @@ install =
location = $${directory:frontend-static}/$${:filename} location = $${directory:frontend-static}/$${:filename}
filename = $${:_buildout_section_name_} filename = $${:_buildout_section_name_}
# Local Python Server
# -------------------
[python-server-port]
recipe = slapos.cookbook:free_port
minimum = 3000
maximum = 3100
ip = {{ ipv4_random }}
[python-server]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${buildout:executable} -m http.server $${python-server-port:port} --bind $${python-server-port:ip} --directory $${directory:frontend-static}
# Common Environment # Common Environment
# ------------------ # ------------------
...@@ -406,7 +480,6 @@ hash-existing-files = ...@@ -406,7 +480,6 @@ hash-existing-files =
ip = {{ ipv4_random }} ip = {{ ipv4_random }}
hostname = $${:ip} hostname = $${:ip}
port = $${theia-service:port} port = $${theia-service:port}
base-url = $${theia-service:base-url}
[theia-shell] [theia-shell]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
......
[buildout] [buildout]
extends = extends =
../../component/caddy/buildout.cfg ../../component/haproxy/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/bash/buildout.cfg ../../component/bash/buildout.cfg
../../component/bash-completion/buildout.cfg ../../component/bash-completion/buildout.cfg
......
...@@ -146,17 +146,17 @@ class TestTheia(TheiaTestCase): ...@@ -146,17 +146,17 @@ class TestTheia(TheiaTestCase):
)).geturl() )).geturl()
self.get(authenticated_url) self.get(authenticated_url)
# there's a public folder to serve file # there's a public folder to serve file (no need for authentication)
with open('{}/srv/frontend-static/public/test_file'.format( with open('{}/srv/frontend-static/public/test_file'.format(
self.getPath()), 'w') as f: self.getPath()), 'w') as f:
f.write("hello") f.write("hello")
resp = self.get(urljoin(authenticated_url, '/public/')) resp = self.get(urljoin(url, '/public/'))
self.assertIn('test_file', resp.text) self.assertIn('test_file', resp.text)
resp = self.get(urljoin(authenticated_url, '/public/test_file')) resp = self.get(urljoin(url, '/public/test_file'))
self.assertEqual('hello', resp.text) self.assertEqual('hello', resp.text)
# there's a (not empty) favicon # there's a (not empty) favicon (no need for authentication)
resp = self.get(urljoin(authenticated_url, '/favicon.ico')) resp = self.get(urljoin(url, '/favicon.ico'))
self.assertTrue(resp.raw) self.assertTrue(resp.raw)
# there is a CSS referencing fonts # there is a CSS referencing fonts
......
...@@ -176,7 +176,7 @@ context = ...@@ -176,7 +176,7 @@ context =
key dash_location dash:location key dash_location dash:location
key bash_location bash:location key bash_location bash:location
key dcron_location dcron:location key dcron_location dcron:location
key default_cloudooo_url erp5-defaults:cloudooo-connection-url key default_cloudooo_url_list erp5-defaults:cloudooo-connection-url-list
key erp5_location erp5:location key erp5_location erp5:location
key findutils_location findutils:location key findutils_location findutils:location
key gzip_location gzip:location key gzip_location gzip:location
...@@ -333,7 +333,11 @@ repository_id_list = erp5 erp5-bin erp5-doc ...@@ -333,7 +333,11 @@ repository_id_list = erp5 erp5-bin erp5-doc
# ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin) # ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin)
[erp5-defaults] [erp5-defaults]
cloudooo-connection-url = https://cloudooo.erp5.net/ # Default cloudooo is https://cloudooo.erp5.net/ and https://cloudooo1.erp5.net/ in random order.
# The random order is applied in instance profile
cloudooo-connection-url-list =
https://cloudooo.erp5.net/
https://cloudooo1.erp5.net/
# Jupyter is by default disabled in ERP5 # Jupyter is by default disabled in ERP5
jupyter-enable-default = false jupyter-enable-default = false
# WCFS is by default disabled in ERP5 # WCFS is by default disabled in ERP5
......
...@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04 ...@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 74979d6743cdc7ce3d2f17680115458d md5sum = 3f7b28085ceff321a3cb785db60f7c3e
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 52dfc0c9a4547b1e928707ef5caeecdb md5sum = 30a1e738a8211887e75a5e75820e5872
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880 ...@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
[template-zope] [template-zope]
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = 1d0cfbefbfee93e28c07f7813b94723c md5sum = 90c67d5a86a42cb0dd00e45fd099b1c2
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -21,6 +21,10 @@ ...@@ -21,6 +21,10 @@
{% set test_runner_total_database_count = mariadb_test_database_amount %} {% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %} {% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%} {% endif -%}
{# Backward compatibility for cloudooo-url #}
{% if slapparameter_dict.get('cloudooo-url') -%}
{% do slapparameter_dict.setdefault('cloudooo-url-list', slapparameter_dict['cloudooo-url'].split(',')) %}
{% endif -%}
{% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%} {% set test_runner_random_activity_priority = slapparameter_dict.get('test-runner', {}).get('random-activity-priority') -%}
{% set monitor_base_url_dict = {} -%} {% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
...@@ -234,7 +238,7 @@ return = ...@@ -234,7 +238,7 @@ return =
{% endif -%} {% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }} config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
config-caucase-url = {{ dumps(caucase_url) }} config-caucase-url = {{ dumps(caucase_url) }}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
......
...@@ -33,7 +33,6 @@ partition. No more (undefined result), no less (IndexError). ...@@ -33,7 +33,6 @@ partition. No more (undefined result), no less (IndexError).
{% for alias, url in ( {% for alias, url in (
('erp5-memcached-volatile', slapparameter_dict['memcached-url']), ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
('erp5-memcached-persistent', slapparameter_dict['kumofs-url']), ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
('erp5-smtp', slapparameter_dict['smtp-url']), ('erp5-smtp', slapparameter_dict['smtp-url']),
) -%} ) -%}
{% set parsed_url = urllib_parse.urlparse(url) -%} {% set parsed_url = urllib_parse.urlparse(url) -%}
...@@ -269,12 +268,7 @@ bt5 = {{ slapparameter_dict['bt5'] }} ...@@ -269,12 +268,7 @@ bt5 = {{ slapparameter_dict['bt5'] }}
bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }} bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }} id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }}
home = ${buildout:directory} home = ${buildout:directory}
# We only want to change the hostname to 'erp5-cloudooo' if we use the internal cloudooo-url = {{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
# cloudooo. We plan to remove the ability to have an internal one, so this
# heuristic is enough.
{% set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
cloudooo-url = {{ (cloudooo if cloudooo.port == None else
cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
{% endif -%} {% endif -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }} developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
...@@ -514,11 +508,8 @@ wrapper-path = ${directory:bin}/${:command-name}.real ...@@ -514,11 +508,8 @@ wrapper-path = ${directory:bin}/${:command-name}.real
command-line = command-line =
'{{ parameter_dict['bin-directory'] }}/${:command-name}' '{{ parameter_dict['bin-directory'] }}/${:command-name}'
${:command-line-extra} ${:command-line-extra}
--conversion_server_url={{ slapparameter_dict['cloudooo-url'] }} --conversion_server_url={{ ",".join(slapparameter_dict['cloudooo-url-list']) }}
--conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }} --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
{#- BBB: We still have test suites that only accept the following 2 options. #}
--conversion_server_hostname=erp5-cloudooo
--conversion_server_port={{ port_dict['erp5-cloudooo'] }}
--volatile_memcached_server_hostname=erp5-memcached-volatile --volatile_memcached_server_hostname=erp5-memcached-volatile
--volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }} --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
--persistent_memcached_server_hostname=erp5-memcached-persistent --persistent_memcached_server_hostname=erp5-memcached-persistent
......
...@@ -26,8 +26,21 @@ extra-context = ...@@ -26,8 +26,21 @@ extra-context =
section parameter_dict dynamic-template-postfix-parameters section parameter_dict dynamic-template-postfix-parameters
import urllib urllib import urllib urllib
[default-cloudooo-url-list]
recipe = slapos.recipe.build
default-url-list =
{{ default_cloudooo_url_list | indent }}
init =
# Expose the default cloudooo URLs, sorted using a pseudo-random seed,
# so that the same instance keep the same order
import six, random
seed = repr(sorted(six.iteritems(self.buildout['slap-connection'])))
default_cloudooo_url_list = [url.strip() for url in self.options['default-url-list'].splitlines()]
random.Random(seed).shuffle(default_cloudooo_url_list)
self.options['url-list'] = default_cloudooo_url_list
[dynamic-template-erp5-parameters] [dynamic-template-erp5-parameters]
default-cloudooo-url = {{ dumps(default_cloudooo_url) }} default-cloudooo-url-list = ${default-cloudooo-url-list:url-list}
jupyter-enable-default = {{ jupyter_enable_default }} jupyter-enable-default = {{ jupyter_enable_default }}
wcfs-enable-default = {{ wcfs_enable_default }} wcfs-enable-default = {{ wcfs_enable_default }}
local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }} local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
...@@ -42,7 +55,7 @@ template-zodb-base = {{ template_zodb_base }} ...@@ -42,7 +55,7 @@ template-zodb-base = {{ template_zodb_base }}
url = {{ template_erp5 }} url = {{ template_erp5 }}
filename = instance-erp5.cfg filename = instance-erp5.cfg
extra-context = extra-context =
key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url key default_cloudooo_url_list dynamic-template-erp5-parameters:default-cloudooo-url-list
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
......
...@@ -18,7 +18,7 @@ md5sum = e24d0567cda7c72e39f6c8fcb0ede7ab ...@@ -18,7 +18,7 @@ md5sum = e24d0567cda7c72e39f6c8fcb0ede7ab
[pbsready-import] [pbsready-import]
filename = pbsready-import.cfg.in filename = pbsready-import.cfg.in
md5sum = 0aa7feb615fc5ac1a132a6d92c80f954 md5sum = da1d3229fe6d3664720d75b2e9d0b8f4
[pbsready-export] [pbsready-export]
filename = pbsready-export.cfg.in filename = pbsready-export.cfg.in
...@@ -46,7 +46,7 @@ md5sum = 2c9828b57a65ee1eda62fcc4e9a92714 ...@@ -46,7 +46,7 @@ md5sum = 2c9828b57a65ee1eda62fcc4e9a92714
[resilient-web-takeover-cgi-script-download] [resilient-web-takeover-cgi-script-download]
filename = resilient-web-takeover-cgi-script.py.in filename = resilient-web-takeover-cgi-script.py.in
md5sum = 4137b7ac6b20a6cd897ad58f51a763b1 md5sum = 8f5c9830f4865265d3477fe51425e71d
[template-wrapper] [template-wrapper]
filename = templates/wrapper.in filename = templates/wrapper.in
......
...@@ -105,7 +105,6 @@ takeover-triggered-file-path = $${resiliency-takeover-script:takeover-triggered- ...@@ -105,7 +105,6 @@ takeover-triggered-file-path = $${resiliency-takeover-script:takeover-triggered-
[resilient-web-takeover-password] [resilient-web-takeover-password]
recipe = slapos.cookbook:generate.password recipe = slapos.cookbook:generate.password
storage-path = $${directory:srv}/passwd storage-path = $${directory:srv}/passwd
bytes = 8
[resilient-web-takeover-cgi-script] [resilient-web-takeover-cgi-script]
recipe = collective.recipe.template recipe = collective.recipe.template
......
...@@ -7,8 +7,8 @@ takeover_script = '${resiliency-takeover-script:wrapper-takeover}' ...@@ -7,8 +7,8 @@ takeover_script = '${resiliency-takeover-script:wrapper-takeover}'
import atexit import atexit
import cgi import cgi
import cgitb
import datetime import datetime
import hmac
try: try:
import dbm.gnu as gdbm import dbm.gnu as gdbm
except ImportError: except ImportError:
...@@ -25,8 +25,6 @@ if os.path.exists('resilient_software_release_information.py'): ...@@ -25,8 +25,6 @@ if os.path.exists('resilient_software_release_information.py'):
else: else:
resilient_main = lambda: {} resilient_main = lambda: {}
cgitb.enable()
def deleteTemporaryDirectory(path): def deleteTemporaryDirectory(path):
if os.path.exists(path): if os.path.exists(path):
shutil.rmtree(path) shutil.rmtree(path)
...@@ -108,7 +106,7 @@ if "password" not in form: ...@@ -108,7 +106,7 @@ if "password" not in form:
</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted())) </html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted()))
sys.exit(0) sys.exit(0)
if form['password'].value != '${:password}': if not hmac.compare_digest(form['password'].value.encode(), b'${:password}'):
print("<H1>Error</H1>") print("<H1>Error</H1>")
print("Password is invalid.") print("Password is invalid.")
sys.exit(1) sys.exit(1)
......
...@@ -152,7 +152,7 @@ backports.shutil-get-terminal-size = 1.0.0 ...@@ -152,7 +152,7 @@ backports.shutil-get-terminal-size = 1.0.0
bcrypt = 3.1.4 bcrypt = 3.1.4
bleach = 5.0.1 bleach = 5.0.1
CacheControl = 0.12.6:whl CacheControl = 0.12.6:whl
certifi = 2022.6.15 certifi = 2022.12.7
cffi = 1.15.0 cffi = 1.15.0
chardet = 3.0.4 chardet = 3.0.4
charset-normalizer = 2.1.1 charset-normalizer = 2.1.1
...@@ -184,8 +184,8 @@ funcsigs = 1.0.2 ...@@ -184,8 +184,8 @@ funcsigs = 1.0.2
functools32 = 3.2.3.post2 functools32 = 3.2.3.post2
gevent = 20.9.0 gevent = 20.9.0
geventmp = 0.0.1 geventmp = 0.0.1
gitdb2 = 2.0.5 gitdb = 4.0.10
GitPython = 2.1.11 GitPython = 3.1.30
greenlet = 0.4.17 greenlet = 0.4.17
h5py = 2.7.1 h5py = 2.7.1
h11 = 0.14.0 h11 = 0.14.0
...@@ -294,7 +294,7 @@ slapos.recipe.cmmi = 0.19 ...@@ -294,7 +294,7 @@ slapos.recipe.cmmi = 0.19
slapos.recipe.template = 5.0 slapos.recipe.template = 5.0
slapos.toolbox = 0.128 slapos.toolbox = 0.128
statsmodels = 0.11.1 statsmodels = 0.11.1
smmap2 = 2.0.5 smmap = 5.0.0
sniffio = 1.3.0 sniffio = 1.3.0
sortedcontainers = 2.4.0 sortedcontainers = 2.4.0
stevedore = 1.21.0:whl stevedore = 1.21.0:whl
...@@ -314,7 +314,7 @@ urllib3 = 1.26.12 ...@@ -314,7 +314,7 @@ urllib3 = 1.26.12
wcwidth = 0.2.5 wcwidth = 0.2.5
webencodings = 0.5.1 webencodings = 0.5.1
Werkzeug = 2.0.2 Werkzeug = 2.0.2
wheel = 0.35.1:whl wheel = 0.38.4:whl
widgetsnbextension = 2.0.0 widgetsnbextension = 2.0.0
wsproto = 1.2.0 wsproto = 1.2.0
xml-marshaller = 1.0.2 xml-marshaller = 1.0.2
...@@ -335,10 +335,16 @@ charset-normalizer = 2.0.12 ...@@ -335,10 +335,16 @@ charset-normalizer = 2.0.12
click = 6.7 click = 6.7
distro = 1.6.0 distro = 1.6.0
feedparser = 5.2.1 feedparser = 5.2.1
gitdb2 = 2.0.5
GitPython = 2.1.11
pyparsing = 2.2.0 pyparsing = 2.2.0
pyrsistent = 0.16.1 pyrsistent = 0.16.1
requests = 2.27.1 requests = 2.27.1
smmap = 0.9.0
smmap2 = 2.0.5
traitlets = 4.3.3
Werkzeug = 1.0.1 Werkzeug = 1.0.1
wheel = 0.35.1:whl
[versions:sys.version_info < (3,8)] [versions:sys.version_info < (3,8)]
MarkupSafe = 1.0 MarkupSafe = 1.0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment