Commit 540e3b64 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents d445ac7f 609a1382
......@@ -23,7 +23,6 @@ extends =
../imagemagick/buildout.cfg
../icu/buildout.cfg
../openssl/buildout.cfg
../zstd/buildout.cfg
[php-redis]
recipe = slapos.recipe.cmmi
......@@ -116,52 +115,6 @@ environment =
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib
TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
HOME=${apache:location}
LD_LIBRARY_PATH=${zstd:location}/lib
[apache-php-postgres]
# Note: Shall react on each build of apache and reinstall itself
recipe = slapos.recipe.cmmi
url = http://fr2.php.net/distributions/php-5.4.12.tar.bz2
md5sum = 5c7b614242ae12e9cacca21c8ab84818
configure-options =
--with-apxs2=${apache:location}/bin/apxs
--with-libxml-dir=${libxml2:location}
--with-zlib-dir=${zlib:location}
--with-bz2=${bzip2:location}
--with-mcrypt=${libmcrypt:location}
--with-gd
--with-jpeg-dir=${libjpeg:location}
--with-png-dir=${libpng:location}
--enable-gd-native-ttf
--with-freetype-dir=${freetype:location}
--with-curl=${curl:location}
--with-imap=${cclient:location}
--with-iconv-dir=${libiconv:location}
--with-gettext=${gettext:location}
--with-ldap=${openldap:location}
--with-imap-ssl
--with-openssl=${openssl:location}
--enable-libxml
--enable-mbstring
--enable-session
--enable-exif
--enable-zip
--enable-ftp
--with-pgsql=${postgresql:location}
# Changing TMPDIR is required for PEAR installation.
# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
# XXX we could mkdir tmp there
environment =
PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
HOME=${apache:location}
[libmcrypt]
......
......@@ -4,7 +4,6 @@ parts =
[noVNC]
recipe = slapos.recipe.build:download-unpacked
# version-1.0.0 release on Feb 22, 2018
url = https://github.com/novnc/noVNC/archive/v1.0.0.tar.gz
md5sum = d63c2944abd12fecec6ec504e82f27c8
url = https://github.com/novnc/noVNC/archive/refs/tags/v1.2.0.tar.gz
md5sum = 290dfabc4ecdd58d62ccb8c34a922962
strip-top-level-dir = true
......@@ -38,7 +38,7 @@ md5sum = df304a8aee87b6f2425241016a48f7a5
[profile-caddy-frontend-configuration]
_update_hash_filename_ = templates/Caddyfile.in
md5sum = 8cdc462956b6b492c14a53f987c0df5c
md5sum = fdf46b1dee6ea6b91b9aa9e322a0530d
[template-not-found-html]
_update_hash_filename_ = templates/notfound.html
......@@ -46,7 +46,7 @@ md5sum = 88af61e7abbf30dc99a1a2526161128d
[template-default-slave-virtualhost]
_update_hash_filename_ = templates/default-virtualhost.conf.in
md5sum = a0ae858a3db8825c22d33d323392f588
md5sum = 13cd08d630cc51666a9f7e469fb6ea52
[template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in
......
......@@ -7,10 +7,10 @@ import {{ slave_configuration_directory }}/*.conf
bind {{ local_ipv4 }}
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
rotate_size 10000000
}
errors {{ error_log }} {
rotate_size 0
rotate_size 10000000
* {{ not_found_file }}
}
}
......@@ -19,10 +19,10 @@ import {{ slave_configuration_directory }}/*.conf
bind {{ local_ipv4 }}
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
rotate_size 10000000
}
errors {{ error_log }} {
rotate_size 0
rotate_size 10000000
* {{ not_found_file }}
}
}
......@@ -38,10 +38,10 @@ https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv
expvar
pprof
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
rotate_size 10000000
}
errors {{ error_log }} {
rotate_size 0
rotate_size 10000000
* {{ not_found_file }}
}
}
......@@ -59,11 +59,11 @@
} {# tls #}
{%- endif %} {#- if tls #}
log / {{ slave_parameter['access_log'] }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
rotate_size 10000000
}
errors {{ slave_parameter['error_log'] }} {
rotate_size 0
rotate_size 10000000
}
{%- if not (slave_parameter['type'] == 'zope' and backend_url) %}
......
......@@ -477,6 +477,16 @@
},
"type": "object"
},
"wcfs": {
"description": "Parameters for wendelin.core filesystem",
"properties": {
"enable": {
"description": "Whether to enable WCFS filesystem and use it to access ZBigArray/ZBigFile data. In WCFS mode wendelin.core clients (Zope/ERP5 processes) share in-RAM cache for in-ZODB data without duplicating it for every client. This cache sharing does not affect correctness as isolation property is continued to be provided to every client.",
"default": false,
"type": "boolean"
}
}
},
"wendelin-core-zblk-fmt": {
"description": "In wendelin.core there are 2 formats for storing data, so called ZBlk0 and ZBlk1. See https://lab.nexedi.com/nexedi/wendelin.core/blob/2e5e1d3d/bigfile/file_zodb.py#L19 for more details.",
"default": "",
......
# Copyright (C) 2021 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import json
import os.path
import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase, setUpModule as _setUpModule
from .test_erp5 import TestPublishedURLIsReachableMixin
# skip tests when software release is built with wendelin.core 1.
def setUpModule():
_setUpModule()
cls = ERP5InstanceTestCase
if not os.path.exists(
os.path.join(
cls.slap.software_directory,
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test Wendelin Core File System
"""
__partition_reference__ = 'wcfs'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wcfs': {'enable': True}})}
def test_wcfs_accessible(self):
"""Verify that wcfs filesystem is basically accessible.
- we can read .wcfs/zurl
- its content is equal to published `serving-zurl`
"""
zurl = json.loads(
self.getComputerPartition('wcfs').getConnectionParameter('_')
)['serving-zurl']
mntpt = lookupMount(zurl)
zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
self.assertEqual(zurl_, zurl)
# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
def lookupMount(zurl):
for line in readfile('/proc/mounts').splitlines():
# <zurl> <mountpoint> fuse.wcfs ...
zurl_, mntpt, typ, _ = line.split(None, 3)
if typ != 'fuse.wcfs':
continue
if zurl_ == zurl:
return mntpt
raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
return f.read()
......@@ -15,4 +15,4 @@
[instance-cfg]
filename = instance.cfg.in
md5sum = cfb389ef50e1d5e359fef4686dac097d
md5sum = c24f3241af0123c9570a60ccc0fb8fdc
[buildout]
parts =
publish-connection-parameter
stat-password
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -49,6 +50,12 @@ recipe = slapos.cookbook:generate.password
storage-path = $${directory:data}/.passwd
username = admin
[stat-password]
recipe = slapos.recipe.template:jinja2
template = inline:
$${admin-password:username}:$${admin-password:passwd}
rendered = $${directory:data}/passwd
[ice-servers.json]
recipe = slapos.recipe.template:jinja2
template = inline:
......
......@@ -22,7 +22,7 @@ eggs =
<= go-git-package
go.importpath = lab.nexedi.com/nexedi/galene
repository = https://lab.nexedi.com/nexedi/galene.git
revision = galene-0.3
revision = galene-0.3.3
[gowork]
install =
......
Tests for html5as-base software release
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.html5asbase'
with open("README.md") as f:
long_description = f.read()
setup(
name=name,
version=version,
description="Test for SlapOS' HTML5AS Base",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class HTML5ASBaseTestCase(SlapOSInstanceTestCase):
"""
Common class for testing html5as base.
It inherit from SlapOSInstanceTestCase which:
* Install the software release.
* Checks it compile without issue.
* Deploy the instance
* Check deployement works and promise pass
For testing the deployement a different testing class will need to be set up
per each variation of parameters the instance needs to be given.
"""
def checkUrlAndGetResponse(self, url):
"""
Common class to check an url and return the response
"""
response = requests.get(url)
self.assertEqual(requests.codes['OK'], response.status_code)
return response
class TestEmptyDeploy(HTML5ASBaseTestCase):
"""
This class test the instance with no parameters.
"""
def test_deploy_with_no_paramater(self):
"""
Get the connection URL and check it is accessible
"""
url = self.requestDefaultInstance().getConnectionParameterDict()['server_url']
response = self.checkUrlAndGetResponse(url)
result = response.text
self.assertEqual("Hello World!\n", result)
\ No newline at end of file
......@@ -45,4 +45,4 @@ md5sum = 1c0ee16966e1fcdb3fd11c09f12ee2b2
[template_instance_replicate]
_update_hash_filename_ = instance_replicate.cfg.in
md5sum = 1b3efe4ddd2419ad1858be33d1248a17
md5sum = 38d1d352307f79c9c99bf2a80a5c76b8
......@@ -28,6 +28,9 @@ config-download_url = ${slap-parameter:download_url}
name = instance-html5as-{{ i }}
config-port = ${slap-parameter:port-{{ i }}}
config-title = ${slap-parameter:title-{{ i }}}
{% if "sla-%s-computer-guid" % i in slapparameter_dict -%}
sla-computer_guid = {{ slapparameter_dict["sla-%s-computer-guid" % i] }}
{% endif -%}
{% endfor %}
[slap-parameter]
......
......@@ -132,3 +132,51 @@ class TestGracefulWithPortChange(HTML5ASTestCase):
self.checkUrlAndGetResponse(url)
class TestReplicateHTML5AS(HTML5ASTestCase):
"""
This class test the instance with the parameter "port"
"""
instance_parameter_dict = {
"port-1": 8088,
"title-1": "Title 1",
}
@classmethod
def getInstanceSoftwareType(cls):
return 'replicate'
@classmethod
def getInstanceParameterDict(cls):
return cls.instance_parameter_dict
def test_replicate_instance(self):
# Check First instance is deployed with proper parameters
connection_parameter_dict = self.computer_partition.getConnectionParameterDict()
url = connection_parameter_dict['instance-1-server_url']
self.assertEqual(urlparse(url).port, 8088)
response = self.checkUrlAndGetResponse(url)
result = response.text
self.assertTrue("<h1>Title 1</h1>" in result)
# Check only one instance is deployed by default
self.assertTrue("instance-2-server_url" not in connection_parameter_dict)
# Update replicate quantity parameter
self.instance_parameter_dict.update({
'replicate-quantity': 2,
'port-2': 8089,
'sla-2-computer_guid': self.slap._computer_id,
"title-2": "Title 314",
})
# Request instance with the one more replicate
self.requestDefaultInstance()
self.slap.waitForInstance(self.instance_max_retry)
# Check the second replicate
connection_parameter_dict = self.requestDefaultInstance().getConnectionParameterDict()
url = connection_parameter_dict['instance-2-server_url']
self.assertEqual(urlparse(url).port, 8089)
response = self.checkUrlAndGetResponse(url)
result = response.text
self.assertTrue("<h1>Title 314</h1>" in result)
......@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 4c5d8c723b6adb96fcac9690c0c900f7
md5sum = 9d0980144f91d10d8a5cee041f385d34
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......
......@@ -1031,11 +1031,13 @@ context =
{% do part_list.append('wipe-disk-device-wrapper') -%}
[wipe-disk-device-wrapper]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:prerm}/slapos_wipe_device_disk
command-line =
dd if=/dev/zero of={{disk_device_path}} bs=4096 count=500k
recipe = slapos.recipe.template:jinja2
template = inline:
{%- for disk_device in disk_device_path.split() %}
dd if=/dev/zero of={{ disk_device }} bs=4096 count=500k
{%- endfor %}
rendered = ${directory:prerm}/slapos_wipe_device_disk
mode = 0700
{% endif -%}
{% do part_list.append('whitelist-firewall') -%}
......
......@@ -127,6 +127,20 @@ class KvmMixin(object):
if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
return running_process_info.replace(hash_value, '{hash}')
def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError):
self.slap.waitForInstance(max_retry=max_retry)
def rerequestInstance(self, parameter_dict, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
return self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=self.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state)
@skipUnlessKvm
class TestInstance(InstanceTestCase, KvmMixin):
......@@ -582,17 +596,7 @@ class FakeImageHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
return
class FakeImageServerMixin(object):
def rerequestInstance(self, parameter_dict, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
return self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=self.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state)
class FakeImageServerMixin(KvmMixin):
def startImageHttpServer(self):
self.image_source_directory = tempfile.mkdtemp()
server = SocketServer.TCPServer(
......@@ -689,10 +693,6 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
self.stopImageHttpServer()
super(InstanceTestCase, self).tearDown()
def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError):
self.slap.waitForInstance(max_retry=max_retry)
def test(self):
partition_parameter_kw = {
self.key: self.test_input % (
......@@ -1270,3 +1270,31 @@ class TestWhitelistFirewallRequestCluster(TestWhitelistFirewallRequest):
}
}
})}
@skipUnlessKvm
class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
__partition_reference__ = 'ddpwdo'
kvm_instance_partition_reference = 'ddpwdo0'
def test(self):
self.rerequestInstance({
'disk-device-path': '/dev/virt0 /dev/virt1',
'wipe-disk-ondestroy': True
})
self.slap.waitForInstance(max_retry=2)
instance_path = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
slapos_wipe_device_disk = os.path.join(
instance_path, 'etc', 'prerm', 'slapos_wipe_device_disk')
# check prerm script, it's trusted that prerm manager really works
self.assertTrue(os.path.exists(slapos_wipe_device_disk))
with open(slapos_wipe_device_disk) as fh:
self.assertEqual(
fh.read().strip(),
r"""dd if=/dev/zero of=/dev/virt0 bs=4096 count=500k
dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
)
self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))
......@@ -80,7 +80,7 @@ md5sum = 7cefc3dd3bd52b42d381de7d7447691f
[nextcloud-app-news]
<= nc-download-app
url = https://github.com/nextcloud/news/releases/download/14.2.2/news.tar.gz
md5sum = d0f4b355a3609c159ed12dfa55ca5109
md5sum = f48d4b5dcbc078131bb86a4ae619da99
[nextcloud-apps]
spreed = ${nextcloud-app-spreed:location}
......
......@@ -14,7 +14,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = f078d72db690805eab90a4577066b920
md5sum = eba73f868a5bbfb942323eadcd88cb16
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -9,6 +9,8 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
......@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
{% endif -%}
{% endfor -%}
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
{% set zope_address_list_id_dict = {} -%}
{% if zope_partition_dict -%}
[request-zope-base]
<= request-common
request-zodb-base
return =
zope-address-list
hosts-dict
......@@ -233,20 +250,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
config-wcfs_enable = {{ wcfs_enable }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......@@ -348,6 +354,17 @@ config-url = ${request-jupyter:connection-url}
{% endif -%}
{%- endif %}
{% if wcfs_enable -%}
{# request WCFS connected to ZODB -#}
{% do root_common.section('request-wcfs') -%}
{{ request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
[request-wcfs]
<= request-common
request-zodb-base
{%- endif %}
[directory]
recipe = slapos.cookbook:mkdirectory
{% if slapparameter_dict.get('shared-certificate-authority-path', '') -%}
......
......@@ -12,6 +12,7 @@ extra-eggs +=
[template]
extra =
# The following list is for SR whose buildout runs only with Python 3.
helloworld ${slapos.test.helloworld-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
......
......@@ -124,6 +124,11 @@ setup = ${slapos-repository:location}/software/helloworld/test/
egg = slapos.test.html5as
setup = ${slapos-repository:location}/software/html5as/test/
[slapos.test.html5as-base-setup]
<= setup-develop-egg
egg = slapos.test.html5asbase
setup = ${slapos-repository:location}/software/html5as-base/test/
[slapos.test.jupyter-setup]
<= setup-develop-egg
egg = slapos.test.jupyter
......@@ -230,6 +235,7 @@ extra-eggs =
${slapos.test.caucase-setup:egg}
${slapos.test.jscrawler-setup:egg}
${slapos.test.html5as-setup:egg}
${slapos.test.html5as-base-setup:egg}
# We don't name this interpreter `python`, so that when we run slapos node
# software, installation scripts running `python` use a python without any
......@@ -278,11 +284,20 @@ context =
tests =
json-schemas ${slapos.cookbook-setup:setup}
# The following list is for SR that work with either Python 2 and 3
# (as main Python). The test egg must supply a URL which depends on
# the version of Python that is used to run the test.
# Due to a bug in the way promises are run, we may also list some Py3-only SR
# here, to check there's no promise issue when slapos node runs with Python 2.
kvm ${slapos.test.kvm-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
###
${:extra}
extra =
# WARNING: This is for SR that only support Python 2.
# You should not add more lines here.
backupserver ${slapos.test.backupserver-setup:setup}
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
......@@ -302,6 +317,7 @@ extra =
caucase ${slapos.test.caucase-setup:setup}
jscrawler ${slapos.test.jscrawler-setup:setup}
html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup}
[versions]
# slapos.core is used from the clone always
......
......@@ -38,7 +38,7 @@ md5sum = bd0ad0b80d2b39189f9665c48f1b3830
[template_nginx_conf]
filename = nginx_conf.in
md5sum = 2b06f7eb9a1d45d250d4b92a944db925
md5sum = a042e52e0594ec0486cff7dc896cb6ce
[template_httpd_conf]
filename = httpd_conf.in
......
......@@ -41,7 +41,7 @@ http {
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Accel-Mapping /private/;
proxy_set_header Host $http_host;
proxy_connect_timeout 200;
proxy_send_timeout 200;
proxy_read_timeout 200;
......@@ -54,7 +54,7 @@ http {
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Accel-Mapping /private/;
proxy_set_header Host $http_host;
proxy_pass http://unix:{{ socket }};
}
......
......@@ -280,6 +280,28 @@ class TestWeb(SlaprunnerTestCase):
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertIn('SlapOS', resp.text)
def test_slaprunner_redirects(self):
# redirects also work as expected. In this test we visit stopAllPartition
# which should redirect to inspectInstance
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['url']
resp = requests.get(
urljoin(url, '/stopAllPartition'),
verify=False,
auth=(parameter_dict['init-user'], parameter_dict['init-password']))
self.assertEqual(resp.status_code, requests.codes.ok)
self.assertEqual(resp.url, urljoin(url, '/inspectInstance'))
# this also works behind a frontend
resp = requests.get(
urljoin(url, '/stopAllPartition'),
verify=False,
allow_redirects=False,
headers={'Host': 'example.com:1234'},
auth=(parameter_dict['init-user'], parameter_dict['init-password']))
self.assertEqual(resp.status_code, requests.codes.found)
self.assertEqual(resp.headers['Location'], 'https://example.com:1234/inspectInstance')
def test_shellinabox(self):
# shellinabox exists at /shellinabox and is password protected
parameter_dict = self.computer_partition.getConnectionParameterDict()
......
......@@ -185,6 +185,7 @@ context =
key instance_common_cfg instance-common:rendered
key jsl_location jsl:location
key jupyter_enable_default erp5-defaults:jupyter-enable-default
key wcfs_enable_default erp5-defaults:wcfs-enable-default
key kumo_location kumo:location
key local_bt5_repository local-bt5-repository:list
key logrotate_location logrotate:location
......@@ -222,7 +223,9 @@ context =
key template_postfix_aliases template-postfix-aliases:target
key template_postfix_main_cf template-postfix-main-cf:target
key template_postfix_master_cf template-postfix-master-cf:target
key instance_wcfs_cfg_in instance-wcfs.cfg.in:target
key template_zeo template-zeo:target
key template_zodb_base template-zodb-base:target
key template_zope template-zope:target
key template_zope_conf template-zope-conf:target
key template_fonts_conf template-fonts-conf:output
......@@ -237,6 +240,9 @@ context =
[template-zeo]
<= download-base
[template-zodb-base]
<= download-base
[template-zope]
<= download-base
link-binary =
......@@ -281,6 +287,9 @@ fontconfig-includes =
[template-rsyslogd-cfg]
<= download-base
[instance-wcfs.cfg.in]
<= download-base
[erp5-bin]
<= erp5
repository = https://lab.nexedi.com/nexedi/erp5-bin.git
......@@ -320,6 +329,8 @@ repository_id_list = erp5 erp5-bin erp5-doc
cloudooo-connection-url = https://cloudooo.erp5.net/
# Jupyter is by default disabled in ERP5
jupyter-enable-default = false
# WCFS is by default disabled in ERP5
wcfs-enable-default = false
[erp5]
recipe = slapos.recipe.build:gitclone
......@@ -558,6 +569,7 @@ scripts =
runzeo
tidstoraged
tidstorage_repozo
wcfs
web_checker_utility
extra-paths =
......
......@@ -70,19 +70,23 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
[template]
filename = instance.cfg.in
md5sum = b5ac16fdeed8863e465e955ba6d1e12a
md5sum = 8bd7f89b7c1e453ecc952659a01347e6
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = ac155f28a096747fc267f32a1cec46e1
md5sum = 79f092c54998918423257f4b676425ea
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zodb-base]
filename = instance-zodb-base.cfg.in
md5sum = bc821f9f9696953b10a03ad7b59a1936
[template-zope]
filename = instance-zope.cfg.in
md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
md5sum = e495f7225a49f61c501ccc496a976d63
[template-balancer]
filename = instance-balancer.cfg.in
......@@ -95,3 +99,7 @@ md5sum = 3f4f7e49c504cbf610fc5dc462713dfc
[template-rsyslogd-cfg]
filename = rsyslogd.cfg.in
md5sum = 5cf0316fdd17a940031e4083bbededd8
[instance-wcfs.cfg.in]
filename = instance-wcfs.cfg.in
md5sum = 945e8e4552a6bdf228b9609567b09399
......@@ -9,6 +9,8 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
......@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
{% endif -%}
{% endfor -%}
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
{% set zope_address_list_id_dict = {} -%}
{% if zope_partition_dict -%}
[request-zope-base]
<= request-common
request-zodb-base
return =
zope-address-list
hosts-dict
......@@ -238,20 +255,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
config-wcfs_enable = {{ wcfs_enable }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......@@ -353,6 +359,17 @@ config-url = ${request-jupyter:connection-url}
{% endif -%}
{%- endif %}
{% if wcfs_enable -%}
{# request WCFS connected to ZODB -#}
{% do root_common.section('request-wcfs') -%}
{{ request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
[request-wcfs]
<= request-common
request-zodb-base
{%- endif %}
{% set balancer_ret_dict = {'monitor-base-url': False} -%}
{% for family in zope_family_dict -%}
{% do balancer_ret_dict.__setitem__(family, False) -%}
......
{# instance that runs WCFS service associated with ZODB storage #}
{% from "instance_zodb_base" import zodb_dict with context %}
{# build zurl to connect to configured ZODB #}
{% if len(zodb_dict) != 1 -%}
{% do assert(False, ("WCFS supports only single ZODB storage", zodb_dict)) -%}
{% endif -%}
{% set db_name, zodb = zodb_dict.popitem() -%}
{% set z = zodb['storage-dict'] -%}
{% if zodb['type'] == 'zeo' -%}
{% set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
{% elif zodb['type'] == 'neo' -%}
{% set zurl = ('neo://%s@%s' % (z.pop('name'), z.pop('master_nodes'))) -%}
{% set argv = [] -%}
{% set i = 0 -%}
{% for k,v in z|dictsort -%}
{% do argv.append('%s=%s' % (k,v)) -%}
{% endfor -%}
{% if len(argv) > 0 -%}
{% set zurl = zurl + '&' + '?'.join(argv) -%}
{% endif -%}
{% else -%}
{% do assert(False, ("unsupported ZODB type", zodb)) -%}
{% endif -%}
[buildout]
extends = {{ template_monitor }}
parts +=
wcfs
wcfs-promise
publish
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
log = ${:var}/log
run = ${:var}/run
services = ${:etc}/run
service-on-watch = ${:etc}/service
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
var = ${buildout:directory}/var
[wcfs]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/wcfs serve -log_dir=${directory:log} {{ zurl }}
wrapper-path = ${directory:service-on-watch}/wcfs
[wcfs-promise]
<= monitor-promise-base
module = check_command_execute
name = ${:_buildout_section_name_}.py
config-command = {{ bin_directory }}/wcfs status {{ zurl }}
[publish]
recipe = slapos.cookbook:publish.serialised
serving-zurl = {{ zurl }}
{# base for instances that need to access ZODB storage #}
{# provides zodb_dict #}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'zeo' -%}
{% do storage_dict.update(zeo_dict.get(name, ())) -%}
{% else -%}
{% if name == slapparameter_dict.get('neo-name') -%}
{% do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
name=slapparameter_dict['neo-cluster']) -%}
{% endif -%}
{{ assert(storage_dict['master_nodes'], name) }}
{% if storage_dict.pop('ssl', 1) -%}
{% do storage_dict.update(ca='~/etc/ca.crt',
cert='~/etc/neo.crt',
key='~/etc/neo.key') -%}
{% endif -%}
{% endif -%}
{% endfor -%}
{% from "instance_zodb_base" import zodb_dict with context %}
{% set wsgi = slapparameter_dict['wsgi'] -%}
{% set webdav = slapparameter_dict['webdav'] -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
{% set site_id = slapparameter_dict['site-id'] -%}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
{% set node_id_base = slapparameter_dict['name'] -%}
{% set selenium_server_configuration_dict = slapparameter_dict.get('selenium-server-configuration-dict', None) -%}
......@@ -63,6 +63,12 @@ environment +=
FONTCONFIG_FILE=${fontconfig-conf:rendered}
{% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
{% endif %}
WENDELIN_CORE_WCFS_AUTOSTART=no
{% if slapparameter_dict['wcfs_enable'] %}
WENDELIN_CORE_VIRTMEM=r:wcfs+w:uvmm
{% else %}
WENDELIN_CORE_VIRTMEM=rw:uvmm
{% endif %}
${:environment-extra}
......@@ -257,24 +263,6 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else
cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
{% endif -%}
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'zeo' -%}
{% do storage_dict.update(zeo_dict.get(name, ())) -%}
{% else -%}
{% if name == slapparameter_dict.get('neo-name') -%}
{% do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
name=slapparameter_dict['neo-cluster']) -%}
{% endif -%}
{{ assert(storage_dict['master_nodes'], name) }}
{% if storage_dict.pop('ssl', 1) -%}
{% do storage_dict.update(ca='~/etc/ca.crt',
cert='~/etc/neo.crt',
key='~/etc/neo.key') -%}
{% endif -%}
{% endif -%}
{% endfor -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
publisher-timeout = {{ dumps(slapparameter_dict['publisher-timeout']) }}
activity-timeout = {{ dumps(slapparameter_dict['activity-timeout']) }}
......
......@@ -31,11 +31,13 @@ extra-context =
[dynamic-template-erp5-parameters]
default-cloudooo-url = {{ dumps(default_cloudooo_url) }}
jupyter-enable-default = {{ jupyter_enable_default }}
wcfs-enable-default = {{ wcfs_enable_default }}
local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
[context]
root-common = {{ root_common }}
caucase-jinja2-library = {{ caucase_jinja2_library }}
template-zodb-base = {{ template_zodb_base }}
[dynamic-template-erp5]
<= jinja2-template-base
......@@ -44,6 +46,7 @@ filename = instance-erp5.cfg
extra-context =
key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
key openssl_location :openssl-location
import re re
......@@ -125,6 +128,8 @@ extra-context =
import urlparse urlparse
import hashlib hashlib
import itertools itertools
import-list =
file instance_zodb_base context:template-zodb-base
[dynamic-template-kumofs-parameters]
<= default-dynamic-template-parameters
......@@ -190,6 +195,15 @@ mode = 644
[dynamic-template-jupyter]
rendered = {{ template_jupyter_cfg }}
[dynamic-template-wcfs]
<= jinja2-template-base
template = {{ instance_wcfs_cfg_in }}
filename = instance_wcfs.cfg
extra-context =
section parameter_dict dynamic-template-zope-parameters
import-list =
file instance_zodb_base context:template-zodb-base
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
override = {{ dumps(override_switch_softwaretype |default) }}
......@@ -208,6 +222,7 @@ zodb-zeo = dynamic-template-zeo:rendered
zodb-neo = neo:rendered
zope = dynamic-template-zope:rendered
jupyter = dynamic-template-jupyter:rendered
wcfs = dynamic-template-wcfs:rendered
# Keep cloudooo backward compatibility
cloudooo = dynamic-template-legacy:output
......
......@@ -208,7 +208,7 @@ setuptools-dso = 1.7
rubygemsrecipe = 0.3.0
six = 1.12.0
slapos.cookbook = 1.0.184
slapos.core = 1.6.8
slapos.core = 1.6.9
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
......@@ -216,7 +216,7 @@ slapos.rebootstrap = 4.5
slapos.recipe.build = 0.46
slapos.recipe.cmmi = 0.17
slapos.recipe.template = 4.5
slapos.toolbox = 0.118
slapos.toolbox = 0.119
stevedore = 1.21.0
subprocess32 = 3.5.4
unicodecsv = 0.14.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment