diff --git a/component/gnupg/buildout.cfg b/component/gnupg/buildout.cfg new file mode 100644 index 0000000000000000000000000000000000000000..d78a3a1527546c042233c4885ed6a6b990f5905f --- /dev/null +++ b/component/gnupg/buildout.cfg @@ -0,0 +1,44 @@ +[buildout] +extends = + ../bzip2/buildout.cfg + ../file/buildout.cfg + ../gnutls/buildout.cfg + ../sqlite3/buildout.cfg +parts = gnupg + +[npth] +<= gpg-common +version = 1.5 +md5sum = 9ba2dc4302d2f32c66737c43ed191b1b +configure-options-extra = + --disable-tests + +[libassuan] +<= with-gpg-error +version = 2.4.3 +md5sum = 8e01a7c72d3e5d154481230668e6eb5a + +[libksba] +<= with-gpg-error +version = 1.3.5 +md5sum = 8302a3e263a7c630aa7dea7d341f07a2 + +[gnupg] +<= libgcrypt +version = 2.2.1 +md5sum = f781efae8756f6cf5d500aad8e4b33e2 +configure-options += + --disable-ldap + --disable-card-support + --disable-photo-viewers + --with-bzip2=${bzip2:location} + --with-ksba-prefix=${libksba:location} + --with-libassuan-prefix=${libassuan:location} + --with-libgcrypt-prefix=${libgcrypt:location} + --with-npth-prefix=${npth:location} + --with-readline=${readline:location} +environment = + PATH=${pkgconfig:location}/bin:${gettext:location}/bin:%(PATH)s + PKG_CONFIG_PATH=${gnutls:pkg-config-path}:${gnutls:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig + CPPFLAGS=-I${file:location}/include -I${libgcrypt:location}/include + LDFLAGS=-L${file:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib -Wl,-rpath=${libgpg-error:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -Wl,-rpath=${libksba:location}/lib -Wl,-rpath=${libassuan:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -Wl,-rpath=${npth:location}/lib diff --git a/component/gnutls/buildout.cfg b/component/gnutls/buildout.cfg index 66f836519ff88092c37d1ce71450f33ab8844baa..7ec241a49ba980ea6e72aeda9a0a047f62faab5c 100644 --- a/component/gnutls/buildout.cfg +++ b/component/gnutls/buildout.cfg @@ -14,7 +14,6 @@ url = http://www.gnupg.org/ftp/gcrypt/${:_buildout_section_name_}/${:_buildout_s configure-options = ${:configure-options-extra} --enable-option-checking=fatal --disable-dependency-tracking - --disable-doc environment = ${:environment-extra} PATH=${gettext:location}/bin:%(PATH)s environment-extra = @@ -24,22 +23,30 @@ environment-extra = version = 1.27 md5sum = 5217ef3e76a7275a2a3b569a12ddc989 configure-options-extra = + --disable-doc --disable-tests -[libgcrypt] +[with-gpg-error] <= gpg-common -version = 1.8.1 -md5sum = b21817f9d850064d2177285f1073ec55 -configure-options-extra = +configure-options-extra = ${:configure-options-extra2} --with-gpg-error-prefix=${libgpg-error:location} +configure-options-extra2 = environment-extra = LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib +[libgcrypt] +<= with-gpg-error +version = 1.8.1 +md5sum = b21817f9d850064d2177285f1073ec55 +configure-options-extra2 = + --disable-doc + [gnutls] <= gpg-common url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.15.tar.xz md5sum = bcdcbc65c50a7499617ad9f4d0058de9 configure-options-extra = + --disable-doc --disable-static --disable-libdane --disable-guile diff --git a/component/golang/buildout.cfg b/component/golang/buildout.cfg index 2911e7cf77140af28a6b55d94ed7ed631d3cf656..33c892e87b38cb0f9d982402a3d9874a61013185 100644 --- a/component/golang/buildout.cfg +++ b/component/golang/buildout.cfg @@ -29,6 +29,7 @@ revision = go1.4.3-16-g4d5426a <= golang-common path = ${golang14-repository:location} environment-extra = +make-targets= cd src && ./make.bash && cp -alf .. ${:location} [golang18] diff --git a/component/openssl/buildout.cfg b/component/openssl/buildout.cfg index 6ec53c434bcd04fb3ff406a75a26e5d73ad96919..fe7e191eb3ed11ff51044ba37a1f39ae0420d2e1 100644 --- a/component/openssl/buildout.cfg +++ b/component/openssl/buildout.cfg @@ -16,8 +16,8 @@ parts = [openssl] recipe = slapos.recipe.cmmi -url = https://www.openssl.org/source/openssl-1.0.2k.tar.gz -md5sum = f965fc0bf01bf882b31314b61391ae65 +url = https://www.openssl.org/source/openssl-1.0.2m.tar.gz +md5sum = 10e9e37f492094b9ef296f68f24a7666 location = ${buildout:parts-directory}/${:_buildout_section_name_} # 'prefix' option to override --openssldir/--prefix (which is useful # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs diff --git a/setup.py b/setup.py index 45a8a669d83f9016582da42497d89734d4aa086a..6b3defecfe87c170080cb1b4c534cac31b37b769 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ setup(name=name, 'hexagonit.recipe.download', 'netaddr', # to manipulate on IP addresses 'setuptools', # namespaces - 'inotifyx', # to watch filesystem changes (used in lockfile) + 'inotifyx', # XXX use pyinotify instead 'lock_file', #another lockfile implementation for multiprocess 'slapos.core', # uses internally 'zc.buildout', # plays with buildout @@ -133,7 +133,6 @@ setup(name=name, 'lamp.static = slapos.recipe.lamp:Static', 'libcloud = slapos.recipe.libcloud:Recipe', 'libcloudrequest = slapos.recipe.libcloudrequest:Recipe', - 'lockfile = slapos.recipe.lockfile:Recipe', 'logrotate = slapos.recipe.logrotate:Recipe', 'logrotate.d = slapos.recipe.logrotate:Part', 'memcached = slapos.recipe.memcached:Recipe', diff --git a/slapos/recipe/generic_mysql/__init__.py b/slapos/recipe/generic_mysql/__init__.py index 5302489a5a121fe8c841bb90d436fe0706bb1457..334b4c9d7c00e3cbaec40eb95490aa6908e48611 100644 --- a/slapos/recipe/generic_mysql/__init__.py +++ b/slapos/recipe/generic_mysql/__init__.py @@ -80,7 +80,7 @@ class Recipe(GenericBaseRecipe): mysql_script_list = [] # user defined functions - udf_registration = "" + udf_registration = "DROP FUNCTION IF EXISTS last_insert_grn_id;\nDROP FUNCTION IF EXISTS mroonga_snippet;\nDROP FUNCTION IF EXISTS mroonga_command;\n" mroonga = self.options.get('mroonga', 'ha_mroonga.so') if mroonga: udf_registration += "CREATE FUNCTION last_insert_grn_id RETURNS " \ diff --git a/slapos/recipe/lockfile.py b/slapos/recipe/lockfile.py deleted file mode 100644 index b27975f50457d880e1e542672635fdb313d5f5f9..0000000000000000000000000000000000000000 --- a/slapos/recipe/lockfile.py +++ /dev/null @@ -1,90 +0,0 @@ -############################################################################## -# -# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. -# -# WARNING: This program as such is intended to be used by professional -# programmers who take the whole responsibility of assessing all potential -# consequences resulting from its eventual inadequacies and bugs -# End users who are looking for a ready-to-use solution with commercial -# guarantees and support are strongly adviced to contract a Free Software -# Service Company -# -# This program is Free Software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 3 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -############################################################################## -import os -import sys -import subprocess - -import inotifyx - -from slapos.recipe.librecipe import GenericBaseRecipe - -class LockFile(object): - - class LockException(Exception): - pass - - def __init__(self, filename, wait=True, exit=False): - self.filename = filename - if wait: - self.callback = lambda: self.waitDeletion() - elif not exit: - self.callback = lambda: self.raiseException() - else: - self.callback = lambda: sys.exit(1) - - def raiseException(self): - raise LockFile.LockException("Not able to lock the file") - - def waitDeletion(self): - inotify_fd = inotifyx.init() - try: - inotifyx.add_watch(inotify_fd, self.filename, inotifyx.IN_DELETE) - inotifyx.get_events(inotify_fd) - except IOError: # add_watch failed - pass - finally: - os.close(inotify_fd) - - self.__enter__() - - def __enter__(self): - try: - # Atomic file acquisition - self._fd = os.open(self.filename, os.O_CREAT | os.O_EXCL) - except OSError: - self.callback() - - def __exit__(self, exc_type, exc_value, traceback): - os.close(self._fd) - os.unlink(self.filename) - -def locked_run(args): - with LockFile(args['filename'], wait=args['wait'], exit=True): - subprocess.check_call([args['binary']]) - -class Recipe(GenericBaseRecipe): - - def install(self): - wrapper = self.createPythonScript(self.options['wrapper'], - __name__ + '.locked_run', - dict( - filename=self.options['lock-file'], - wait=self.optionIsTrue('wait', False), - binary=self.options['binary'], - ) - ) - return [wrapper] diff --git a/software/apache-frontend/buildout.hash.cfg b/software/apache-frontend/buildout.hash.cfg index 4c49d74f7628d679022707d7dc1b107df9c5beed..77135a2e7358e0f4c3a7559ead794da2ae503f8f 100644 --- a/software/apache-frontend/buildout.hash.cfg +++ b/software/apache-frontend/buildout.hash.cfg @@ -39,7 +39,7 @@ md5sum = 665e83d660c9b779249b2179d7ce4b4e [template-apache-frontend-configuration] filename = templates/apache.conf.in -md5sum = 82cdb4ab02fec36285b9c1ce502f82f0 +md5sum = a56045e7b53ff00ab34d2a8f911fc1a1 [template-custom-slave-list] filename = templates/apache-custom-slave-list.cfg.in diff --git a/software/apache-frontend/templates/apache.conf.in b/software/apache-frontend/templates/apache.conf.in index 05b241a2a8a5e11a86b48a8687112082f820fd73..f7083bd1ee4d58674e91c08079e4947547f753f2 100644 --- a/software/apache-frontend/templates/apache.conf.in +++ b/software/apache-frontend/templates/apache.conf.in @@ -166,8 +166,8 @@ SSLProxyCheckPeerCN off SSLProxyCheckPeerExpire off include {{frontend_configuration.get('log-access-configuration')}} -include {{ slave_configuration_directory }}/*.conf -include {{ slave_with_cache_configuration_directory }}/*.conf +includeoptional {{ slave_configuration_directory }}/*.conf +includeoptional {{ slave_with_cache_configuration_directory }}/*.conf ErrorDocument 404 /notfound.html RewriteRule (.*) /notfound.html [R=404,L] diff --git a/software/buildout-testing/runTestSuite.in b/software/buildout-testing/runTestSuite.in new file mode 100644 index 0000000000000000000000000000000000000000..40e6895af594eac599ca6deb694e524f3df05b12 --- /dev/null +++ b/software/buildout-testing/runTestSuite.in @@ -0,0 +1,128 @@ +#!{{runTestSuite_py}} +from __future__ import print_function +import argparse, os, re, subprocess, sys +from time import gmtime, strftime, time +from erp5.util import taskdistribution +from erp5.util.testsuite import SubprocessError, TestSuite +from zc.buildout.buildout import Buildout + +slapos_buildout = {{repr(slapos_buildout)}} +test_dict = { + 'zc.buildout': slapos_buildout, + 'zc.recipe.egg': os.path.join(slapos_buildout, 'zc.recipe.egg_'), +} + +class DummyTestResult: + + class DummyTestResultLine: + def stop(self, **kw): + pass + + done = 0 + + def __init__(self, test_name_list): + self.test_name_list = test_name_list + + def start(self): + test_result_line = self.DummyTestResultLine() + try: + test_result_line.name = self.test_name_list[self.done] + except IndexError: + return + self.done += 1 + return test_result_line + +class BuildoutTestSuite(TestSuite): + + RUN_RE = re.compile( + r'Ran (?P<all_tests>\d+) tests with' + ' (?P<failures>\d+) failures,' + ' (?P<errors>\d+) errors and' + ' (?P<skips>\d+) skipped in') + + def run(self, test): + start = time() + try: + status_dict = self.spawn(os.path.join('bin', 'zope-testrunner'), + '--test-path', os.path.join(test_dict[test], 'src')) + except SubprocessError, e: + status_dict = e.status_dict + end = time() + status_dict.update( + date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)), + duration = end - start) + search = self.RUN_RE.search(status_dict['stdout']) + if search: + groupdict = search.groupdict() + status_dict.update( + test_count = int(groupdict['all_tests']), + error_count = int(groupdict['errors']), + failure_count = int(groupdict['failures']), + skip_count = int(groupdict['skips'])) + return status_dict + +def main(): + os.environ['TEMP'] = {{repr(temp_directory)}} + parser = argparse.ArgumentParser(description='Run a test suite.') + parser.add_argument('--test_suite', help='The test suite name') + parser.add_argument('--test_suite_title', help='The test suite title') + parser.add_argument('--test_node_title', help='The test node title') + parser.add_argument('--project_title', help='The project title') + parser.add_argument('--revision', help='The revision to test', + default='dummy_revision') + parser.add_argument('--master_url', + help='The Url of Master controling many suites') + + args = parser.parse_args() + + test_title = args.test_suite_title or args.test_suite + if args.master_url: + tool = taskdistribution.TaskDistributionTool(args.master_url) + test_result = tool.createTestResult(args.revision, + list(test_dict), + args.test_node_title, + test_title=test_title, + project_title=args.project_title) + if test_result is None: + return + else: + test_result = DummyTestResult(list(test_dict)) + + fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0666) + try: + os.write(fd, """\ +[buildout] +extends = %s +develop =%s +parts = test + +[versions] +%s + +[bootstrap] +recipe = zc.recipe.egg +eggs = zc.buildout + +[test] +recipe = zc.recipe.egg +eggs += + zope.testrunner +scripts = + zope-testrunner +""" % (os.path.join(slapos_buildout, 'buildout.cfg'), + ''.join('\n ' + x for x in test_dict.itervalues()), + '\n'.join(x + ' =' for x in test_dict))) + finally: + os.close(fd) + Buildout('buildout.cfg', {}).install(['bootstrap']) + subprocess.check_call((os.path.join('bin', 'buildout'),)) + + test_suite = BuildoutTestSuite(1) + while 1: + test_result_line = test_result.start() + if not test_result_line: + break + test_result_line.stop(**test_suite.run(test_result_line.name)) + +if __name__ == "__main__": + main() diff --git a/software/buildout-testing/software-next.cfg b/software/buildout-testing/software-next.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1cd29e9b8cbbafe78e551a8cbe15672eff550fc9 --- /dev/null +++ b/software/buildout-testing/software-next.cfg @@ -0,0 +1,5 @@ +[buildout] +extends = software.cfg + +[versions] +setuptools = 36.6.0 diff --git a/software/buildout-testing/software.cfg b/software/buildout-testing/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..7d758036b1ece13d1ffe9905a57d0a3ab20b44e7 --- /dev/null +++ b/software/buildout-testing/software.cfg @@ -0,0 +1,59 @@ +[buildout] +extends = + ../../stack/slapos.cfg +parts = + slapos-cookbook + template + +[slapos.buildout-repository] +recipe = slapos.recipe.build:gitclone +repository = https://lab.nexedi.com/nexedi/slapos.buildout.git +git-executable = ${git:location}/bin/git + +[runTestSuite_py] +recipe = zc.recipe.egg +eggs = erp5.util + zc.buildout +scripts = ${:interpreter} +interpreter = ${:_buildout_section_name_} + +[template] +recipe = slapos.recipe.template:jinja2 +# XXX: "template.cfg" is hardcoded in instanciation recipe +rendered = ${buildout:directory}/template.cfg +template = + inline: + [buildout] + eggs-directory = ${buildout:eggs-directory} + develop-eggs-directory = ${buildout:develop-eggs-directory} + parts = runTestSuite + + [slap-configuration] + recipe = slapos.cookbook:slapconfiguration.serialised + computer = $${slap-connection:computer-id} + partition = $${slap-connection:partition-id} + url = $${slap-connection:server-url} + key = $${slap-connection:key-file} + cert = $${slap-connection:cert-file} + + [directory] + recipe = slapos.cookbook:mkdirectory + bin = $${buildout:directory}/bin + tmp = $${buildout:directory}/tmp + + [slapos.buildout-repository] + recipe = slapos.recipe.build:gitclone + repository = ${slapos.buildout-repository:location} + git-executable = ${git:location}/bin/git + shared = true + + [runTestSuite] + recipe = slapos.recipe.template:jinja2 + rendered = $${directory:bin}/$${:_buildout_section_name_} + template = ${:_profile_base_location_}/$${:_buildout_section_name_}.in + mode = 0755 + context = + key slapparameter_dict slap-configuration:configuration + key slapos_buildout slapos.buildout-repository:location + key temp_directory directory:tmp + raw runTestSuite_py ${buildout:bin-directory}/${runTestSuite_py:interpreter} diff --git a/software/erp5/instance-erp5-input-schema.json b/software/erp5/instance-erp5-input-schema.json index 3bc4dce75e9a25585b0fd2455e893c98650ba66a..241a9c5a17822c7b3b22d84f1c32c70022576cdf 100644 --- a/software/erp5/instance-erp5-input-schema.json +++ b/software/erp5/instance-erp5-input-schema.json @@ -21,6 +21,10 @@ "description": "Business Template to install at automatic site creation. By default, all configurators are installed.", "type": "string" }, + "id-store-interval": { + "description": "Set Store Interval of default SQL Non Continuous Increasing Id Generator at automatic site creation. If unset, the value from the erp5_core Business Template is not touched.", + "type": "integer" + }, "timezone": { "description": "Zope's timezone. Possible values are determined by host's libc, and typically come from a separate package (tzdata, ...)", "default": "UTC", diff --git a/software/erp5testnode/software.cfg b/software/erp5testnode/software.cfg index 1fae9e4a7bfd6f2b25ef00b4b6f8b2e5a9b874b7..135ce82c5d690d7b443c587305b4e885fd33f5da 100644 --- a/software/erp5testnode/software.cfg +++ b/software/erp5testnode/software.cfg @@ -27,10 +27,9 @@ eggs = zc.buildout slapos.libnetworkcache slapos.core + supervisor jsonschema hexagonit.recipe.download - netaddr - inotifyx lock_file pytz erp5.util diff --git a/software/slapos-in-partition/common.cfg b/software/slapos-in-partition/common.cfg index 1106fd4784d1393ffd0083bc161f2b4c9f5c87d7..8099f97bdd7bd1019cd47525646e9b72e4baec9d 100644 --- a/software/slapos-in-partition/common.cfg +++ b/software/slapos-in-partition/common.cfg @@ -71,21 +71,3 @@ mode = 0644 recipe = zc.recipe.egg eggs = collective.recipe.template - -# Add slapos.libnetworkcache to path of slapos.core. -[slapos-cookbook] -eggs = - ${lxml-python:egg} - slapos.cookbook - cliff - hexagonit.recipe.download - inotifyx - netaddr - netifaces - requests - slapos.core - supervisor - xml_marshaller - pytz - slapos.libnetworkcache - diff --git a/software/slapos-master/buildout.hash.cfg b/software/slapos-master/buildout.hash.cfg index eee921d2b8e6c79c7beaa4fe3e3432bac6999009..695d9001c6ca3aafd8ef24a8e1d4c42bd190ecf9 100644 --- a/software/slapos-master/buildout.hash.cfg +++ b/software/slapos-master/buildout.hash.cfg @@ -15,7 +15,7 @@ # not need these here). [template-erp5] filename = instance-erp5.cfg.in -md5sum = 95f7341ff490cb70df61fad35d08bc30 +md5sum = 251c1fcec1817fe70ec0cf7da69ccf3a [template-balancer] filename = instance-balancer.cfg.in diff --git a/software/slapos-master/instance-erp5.cfg.in b/software/slapos-master/instance-erp5.cfg.in index 647f6b72552986e07dfafc277ddbf1bb679fdd3a..7a2b64173d337b23fd6ee798ea9a991356876961 100644 --- a/software/slapos-master/instance-erp5.cfg.in +++ b/software/slapos-master/instance-erp5.cfg.in @@ -155,6 +155,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }} config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }} +config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }} config-inituser-login = {{ dumps(inituser_login) }} config-inituser-password = ${publish-early:inituser-password} config-kumofs-url = ${request-memcached-persistent:connection-url} diff --git a/software/slaprunner/buildout.hash.cfg b/software/slaprunner/buildout.hash.cfg index bf6a51efc8396a9e22171c39078214941621d9d8..520617bc867b4fb512a3fb2fc3a651b0897ba2d8 100644 --- a/software/slaprunner/buildout.hash.cfg +++ b/software/slaprunner/buildout.hash.cfg @@ -19,11 +19,11 @@ md5sum = 713db528880282d568278f09458d2aab [template-runner] filename = instance-runner.cfg -md5sum = baf70a9f2f433ea0f2f5139700e53b1f +md5sum = 7df86928aeef0fbada832b0283b9a454 [template-runner-import-script] filename = template/runner-import.sh.jinja2 -md5sum = 130193114cbbcd014af9704851410605 +md5sum = 1edd9c7a20e208b6cb647886bfb6d1bb [instance-runner-import] filename = instance-runner-import.cfg.in @@ -55,7 +55,7 @@ md5sum = 525e37ea8b2acf6209869999b15071a6 [template-slapos-cfg] filename = template/slapos.cfg.in -md5sum = ef16446d432e1397182b1654fe920ffb +md5sum = 035e027e9cb9bbdca0509ac895fc4696 [template-parameters] filename = parameters.xml.in diff --git a/software/slaprunner/instance-runner.cfg b/software/slaprunner/instance-runner.cfg index 827ab24b9db59a683622b1495cde0cc645dd6751..fde23abbd3911dae273d520feb305616002917f8 100644 --- a/software/slaprunner/instance-runner.cfg +++ b/software/slaprunner/instance-runner.cfg @@ -137,6 +137,8 @@ working-directory = $${runnerdirectory:home} project-directory = $${runnerdirectory:project} instance_root = $${runnerdirectory:instance-root} software_root = $${runnerdirectory:software-root} +pidfile-software = $${directory:run}/slapgrid-cp.pid +pidfile-instance = $${directory:run}/slapgrid-sr.pid ssh_client = ${openssh:location}/bin/ssh public_key = $${runner-sshd-raw-server:rsa-keyfile}.pub private_key = $${runner-sshd-raw-server:rsa-keyfile} @@ -728,14 +730,12 @@ ip = $${slaprunner:ipv4} server = $${:ip}:$${:port} port = 39986 slapgrid-cp = slapgrid-cp -slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-cp-pid} --verbose --logfile $${:slapgrid-cp-log} +slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${:slapos-cfg} --verbose --logfile $${:slapgrid-cp-log} slapgrid-cp-log = $${runnerdirectory:home}/instance.log -slapgrid-cp-pid = $${directory:run}/slapgrid-cp.pid slapgrid-cp-startretries = 0 slapgrid-sr = slapgrid-sr -slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-sr-pid} --verbose --logfile $${:slapgrid-sr-log} +slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${:slapos-cfg} --verbose --logfile $${:slapgrid-sr-log} slapgrid-sr-log = $${runnerdirectory:home}/software.log -slapgrid-sr-pid = $${directory:run}/slapgrid-sr.pid slapgrid-sr-startretries = 0 slapproxy = slapproxy slapproxy-autorestart = true diff --git a/software/slaprunner/template/runner-import.sh.jinja2 b/software/slaprunner/template/runner-import.sh.jinja2 index a47a0bf357df8bb981bf04388fc087e5e4e8985a..305989ddbb88edb576ed10a44e1edaa037e78605 100644 --- a/software/slaprunner/template/runner-import.sh.jinja2 +++ b/software/slaprunner/template/runner-import.sh.jinja2 @@ -155,22 +155,20 @@ rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true SLAPOSCFG='{{ supervisord["slapos-cfg"] }}' SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}' -SLAPGRIDSRPID='{{ supervisord["slapgrid-sr-pid"] }}' SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}' -SLAPGRIDCPPID='{{ supervisord["slapgrid-cp-pid"] }}' log_message "Building newest Software Release..." -"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 || -"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 || -"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 || +"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || +"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || +"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || (tail -n 200 "$SLAPGRIDSRLOG" && false) # Remove defined scripts to force buildout to recreate them to have updated paths rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true log_message "Fixing Instances as needed after import..." # XXX hardcoded -"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 || -"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 || -"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 || +"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || +"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || +"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || (tail -n 200 "$SLAPGRIDCPLOG" && false) # Invoke defined scripts for each partition inside of slaprunner diff --git a/software/slaprunner/template/slapos.cfg.in b/software/slaprunner/template/slapos.cfg.in index d7c650526595c6c80b6724e908aed5da1594f71e..592c57531b7838d1913acbefeb07f7f2c0f69c1a 100644 --- a/software/slaprunner/template/slapos.cfg.in +++ b/software/slaprunner/template/slapos.cfg.in @@ -6,6 +6,8 @@ computer_id = slaprunner maximal_delay = 0 root_check = {{ slaprunner['root_check'] }} forbid_supervisord_automatic_launch = true +pidfile_software = {{slaprunner['pidfile-software']}} +pidfile_instance = {{slaprunner['pidfile-instance']}} [slapformat] partition_amount = {{ slaprunner['partition-amount'] }} diff --git a/software/test-slave-instance-deployment/software.cfg b/software/test-slave-instance-deployment/software.cfg index 118cc2824639b83a4f7b1297797fde58f85eaf15..c67f8703df5280460980410d0bcb73bbc14b6753 100644 --- a/software/test-slave-instance-deployment/software.cfg +++ b/software/test-slave-instance-deployment/software.cfg @@ -106,23 +106,11 @@ versions = versions networkcache-section = networkcache # Install slapos.cookbook containing all officials recipes -# Explicitely define dependencies as well, because of buildout limitation -# if using "develop" [slapos-cookbook] recipe = zc.recipe.egg eggs = ${lxml-python:egg} slapos.cookbook - cliff - hexagonit.recipe.download - inotifyx - netaddr - netifaces - requests - slapos.core - supervisor - xml_marshaller - pytz [versions] # Use SlapOS patched zc.buildout diff --git a/software/wendelin-scalability/test-fluentd-1pb.cfg b/software/wendelin-scalability/test-fluentd-1pb.cfg new file mode 100644 index 0000000000000000000000000000000000000000..356a9d72a1f2f01f58f319c792422923179d105d --- /dev/null +++ b/software/wendelin-scalability/test-fluentd-1pb.cfg @@ -0,0 +1,45 @@ +[buildout] +extends = test-fluentd-common.cfg + +[template-erp5-patched] +extra = + {%- for family_name, zope_section_id_list in zope_family_dict.items() %} + {%- for zope_section_id in zope_section_id_list %} + [{{zope_section_id}}] + config-family = {{family_name}} + {%- endfor %} + {%- endfor %} + +[template-zope] +recipe = +target = ${template-zope-patched:location} + +[template-zope-base] +<= template-zope +recipe = slapos.recipe.build:download + +[template-zope-patched] +<= template-fluentd +base = ${template-zope-base:target} +method = unpack +tags = + {%- if slapparameter_dict['family'] == 'fluentd' %} + {%- for i, zope in enumerate(publish_list) %} + {%- do tags.__setitem__( + node_id_base ~ (node_id_index_format % i) ~ '-stream', zope[0]) %} + {%- endfor %} + {%- endif %} + +[feeder] +feeder = + # + import os, struct + from random import lognormvariate + pack = struct.Struct('!d').pack + size = 65536 + data = os.urandom(size - 8) + while True: + emit('', + ''.join((pack(time.time()) + data[:int(lognormvariate(10, 1))] + ).ljust(size, '\0') for i in xrange(16))) + time.sleep(interval) diff --git a/software/wendelin-scalability/test-fluentd-common.cfg b/software/wendelin-scalability/test-fluentd-common.cfg new file mode 100644 index 0000000000000000000000000000000000000000..19822ef8e43e9b5b0ce1700bb36152531fb4f6e7 --- /dev/null +++ b/software/wendelin-scalability/test-fluentd-common.cfg @@ -0,0 +1,132 @@ +[buildout] +extends = + ../../component/gnupg/buildout.cfg + ../fluentd/software.cfg + ../wendelin/software.cfg + ../../component/mariadb/mariarocks.cfg + +[local-bt5-repository] +list += ${slapos.cookbook-repository:location}/software/wendelin-scalability + +[patch-template] +recipe = slapos.recipe.build +location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in +script = + with open(self.options['location'], 'w') as dst, \ + open(self.options['base']) as src: + src = src.read() + i = src.index('[buildout]') + dst.write(src[:i] + self.options['extra'] + '\n' + src[i:]) + +[template-erp5] +recipe = +target = ${template-erp5-patched:location} + +[template-erp5-base] +<= template-erp5 +recipe = slapos.recipe.build:download + +[template-erp5-patched] +<= patch-template +base = ${template-erp5-base:target} + +[template-fluentd] +<= patch-template +extra = + {%- set tags = {} %} + ${:tags} + {%- if tags %} + [feeder] + recipe = slapos.cookbook:wrapper + wrapper-path = $${directory:services}/$${:_buildout_section_name_} + command-line = ${feeder:bin-directory}/feeder $${fluentd-agent-conf:rendered} + $${:_buildout_section_name_} .4 + {%- for tag in tags %} + [{{ section(tag) }}] + <= feeder + {%- endfor %} + + [{{ section('fluentd') }}] + recipe = slapos.cookbook:wrapper + wrapper-path = $${directory:services}/$${:_buildout_section_name_} + command-line = ${fluentd:location}/bin/fluentd + -v -c $${fluentd-agent-conf:rendered} + environment = + GEM_PATH=${fluentd:location}/lib/ruby/gems/1.8 + + [fluentd-agent-conf] + recipe = slapos.recipe.template:jinja2 + rendered = $${directory:etc}/fluentd-agent.conf + template = + inline:{% raw -%} + <match debug.**> + @type stdout + @id stdout_output + </match> + <source> +# in_exec is unusable because it logs dropped events in case of buffer overflow + @type unix + path $${directory:run}/fluentd.sock + </source> + {%- for tag, netloc in tags.iteritems() %} + <match {{ tag }}> + @type wendelin + @id {{ tag }} + streamtool_uri http://{{ netloc }}/erp5/portal_ingestion_policies/scalability_test_${:method} + user {{ user }} + password {{ password }} + buffer_type memory + buffer_queue_limit 4 + </match> + {%- endfor %} + {%- endraw %} + context = + key tags :tags + raw user {{ slapparameter_dict['inituser-login'] }} + raw password {{ slapparameter_dict['inituser-password'] }} + tags = {{ dumps(tags) }} + {%- endif %} + +[feeder] +recipe = zc.recipe.egg +eggs = fluent-logger +entry-points = ${:_buildout_section_name_}=__main__:main +initialization = + import re, time, urlparse + from urllib import FancyURLopener + from fluent.sender import FluentSender + def main(): + conf, tag, interval = sys.argv[1:] + with open(conf) as f: + for k, v in re.compile(r"<(.+?)>\s*(.*?)\s*</.+?>", + re.S).findall(f.read()): + v = dict(v.split() for v in v.splitlines()) + if k == 'source': + sock = v['path'] + elif v['@type'] == 'wendelin': + if k.split(None, 1)[1] == tag: + conf = v + emit = FluentSender(tag, host='unix://'+sock, timeout=None, + msgpack_kwargs={'use_bin_type': True}).emit + url = urlparse.urlsplit(conf['streamtool_uri']) + url = urlparse.urlunsplit(url._replace( + netloc='%s:%s@%s' % (conf['user'], conf['password'], url.netloc), + path='/erp5/isPortalBeingCreated')) + urlopen = FancyURLopener().open + while True: + try: + f = urlopen(url) + except IOError: + pass + else: + try: + if not f.read(): + break + finally: + f.close() + time.sleep(5) + interval = float(interval) + ${:feeder} + +[versions] +fluent-logger = 0.5.3 diff --git a/software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg b/software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg new file mode 100644 index 0000000000000000000000000000000000000000..2a537a649082ae26be347ff8b0cb744b365f6a24 --- /dev/null +++ b/software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg @@ -0,0 +1,65 @@ +[buildout] +extends = test-fluentd-common.cfg + +[template-erp5-patched] +extra = + [request-balancer] + config-inituser-login = {{ dumps(inituser_login) }} + config-inituser-password = $${publish-early:inituser-password} + +[template-balancer] +recipe = +target = ${template-balancer-patched:location} + +[template-balancer-base] +<= template-balancer +recipe = slapos.recipe.build:download + +[template-balancer-patched] +<= template-fluentd +base = ${template-balancer-base:target} +method = raw_lf +tags = + {%- set port, backend_list = haproxy_dict['fluentd'] %} + {%- for i in range(len(backend_list)) %} + {%- do tags.__setitem__('wendelin_tag_' ~ i, ipv4 ~ ':' ~ port) %} + {%- endfor %} + +[data.gpg] +recipe = slapos.recipe.build:download +url = http://download.shacache.org/0f56a59ab3bc3193e70c1996fe69260e1b225bc53bbb7509cb3ae036006653dec69b3384cd624eefed81c85e692e4b4b7cb3d25c6244c298070569dddf7216b1 +md5sum = 374e00dfbd28f27819ccfff31d169c9c + +[gpg-decrypt] +recipe = slapos.recipe.build +gpg = ${gnupg:location}/bin/gpg +script = + import subprocess, zc.buildout + options = self.options + args = (options['gpg'], '-d', '--batch', '--passphrase-fd', '0', + '-o', options['location'], options['input']) + p = subprocess.Popen(args, stdin=subprocess.PIPE) + p.communicate(options['password']) + retcode = p.poll() + if retcode: + raise subprocess.CalledProcessError(retcode, ' '.join(args)) + +[data] +<= gpg-decrypt +location = ${data.gpg:location}/data.log +input = ${data.gpg:target} + +[feeder] +feeder = + # + import mmap, os, random + data = mmap.mmap(os.open('${data:location}', os.O_RDONLY), + 0, prot=mmap.PROT_READ) + size = 1024 * 1024 + max_seek = len(data) - size + while True: + a = random.randint(0, max_seek) + a = {'data': data[a:a+size]} + for i in xrange(20): + emit('', a) + time.sleep(interval) diff --git a/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_raw_lf.xml b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_raw_lf.xml new file mode 100644 index 0000000000000000000000000000000000000000..fb9e08ad810e192cb7175a4b282a347d1e4a3780 --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_raw_lf.xml @@ -0,0 +1,151 @@ +<?xml version="1.0"?> +<ZopeData> + <record id="1" aka="AAAAAAAAAAE="> + <pickle> + <global name="Ingestion Policy" module="erp5.portal_type"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>_Access_contents_information_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Auditor</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_Add_portal_content_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_Modify_portal_content_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_View_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Auditor</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>id</string> </key> + <value> <string>scalability_test_raw_lf</string> </value> + </item> + <item> + <key> <string>portal_type</string> </key> + <value> <string>Ingestion Policy</string> </value> + </item> + <item> + <key> <string>script_id</string> </key> + <value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value> + </item> + <item> + <key> <string>workflow_history</string> </key> + <value> + <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent> + </value> + </item> + </dictionary> + </pickle> + </record> + <record id="2" aka="AAAAAAAAAAI="> + <pickle> + <global name="PersistentMapping" module="Persistence.mapping"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>data</string> </key> + <value> + <dictionary> + <item> + <key> <string>validation_workflow</string> </key> + <value> + <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent> + </value> + </item> + </dictionary> + </value> + </item> + </dictionary> + </pickle> + </record> + <record id="3" aka="AAAAAAAAAAM="> + <pickle> + <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> + </pickle> + <pickle> + <tuple> + <none/> + <list> + <dictionary> + <item> + <key> <string>action</string> </key> + <value> <string>validate</string> </value> + </item> + <item> + <key> <string>actor</string> </key> + <value> <string>zope</string> </value> + </item> + <item> + <key> <string>comment</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>error_message</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>time</string> </key> + <value> + <object> + <klass> + <global name="DateTime" module="DateTime.DateTime"/> + </klass> + <tuple> + <none/> + </tuple> + <state> + <tuple> + <float>1507057895.87</float> + <string>UTC</string> + </tuple> + </state> + </object> + </value> + </item> + <item> + <key> <string>validation_state</string> </key> + <value> <string>validated</string> </value> + </item> + </dictionary> + </list> + </tuple> + </pickle> + </record> +</ZopeData> diff --git a/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_unpack.xml b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_unpack.xml new file mode 100644 index 0000000000000000000000000000000000000000..e773f368bfbc41cb8a49c9fdbaa58970a8c87545 --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_unpack.xml @@ -0,0 +1,163 @@ +<?xml version="1.0"?> +<ZopeData> + <record id="1" aka="AAAAAAAAAAE="> + <pickle> + <global name="Ingestion Policy" module="erp5.portal_type"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>_Access_contents_information_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Auditor</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_Add_portal_content_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_Modify_portal_content_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_View_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Auditor</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>description</string> </key> + <value> + <none/> + </value> + </item> + <item> + <key> <string>id</string> </key> + <value> <string>scalability_test_unpack</string> </value> + </item> + <item> + <key> <string>portal_type</string> </key> + <value> <string>Ingestion Policy</string> </value> + </item> + <item> + <key> <string>script_id</string> </key> + <value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value> + </item> + <item> + <key> <string>title</string> </key> + <value> + <none/> + </value> + </item> + <item> + <key> <string>workflow_history</string> </key> + <value> + <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent> + </value> + </item> + </dictionary> + </pickle> + </record> + <record id="2" aka="AAAAAAAAAAI="> + <pickle> + <global name="PersistentMapping" module="Persistence.mapping"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>data</string> </key> + <value> + <dictionary> + <item> + <key> <string>validation_workflow</string> </key> + <value> + <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent> + </value> + </item> + </dictionary> + </value> + </item> + </dictionary> + </pickle> + </record> + <record id="3" aka="AAAAAAAAAAM="> + <pickle> + <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> + </pickle> + <pickle> + <tuple> + <none/> + <list> + <dictionary> + <item> + <key> <string>action</string> </key> + <value> <string>validate</string> </value> + </item> + <item> + <key> <string>actor</string> </key> + <value> <string>zope</string> </value> + </item> + <item> + <key> <string>comment</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>error_message</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>time</string> </key> + <value> + <object> + <klass> + <global name="DateTime" module="DateTime.DateTime"/> + </klass> + <tuple> + <none/> + </tuple> + <state> + <tuple> + <float>1507057913.93</float> + <string>UTC</string> + </tuple> + </state> + </object> + </value> + </item> + <item> + <key> <string>validation_state</string> </key> + <value> <string>validated</string> </value> + </item> + </dictionary> + </list> + </tuple> + </pickle> + </record> +</ZopeData> diff --git a/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.py b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.py new file mode 100644 index 0000000000000000000000000000000000000000..a0e9002a52fdc7ec46a254b8c42d3d6255bbee6f --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.py @@ -0,0 +1,4 @@ +size = 0 +for ds in context.objectValues(): + size += ds.getSize() +return size diff --git a/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml new file mode 100644 index 0000000000000000000000000000000000000000..d5e0948be297f837658914455962e5c5d3cde0cf --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml @@ -0,0 +1,62 @@ +<?xml version="1.0"?> +<ZopeData> + <record id="1" aka="AAAAAAAAAAE="> + <pickle> + <global name="PythonScript" module="Products.PythonScripts.PythonScript"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>Script_magic</string> </key> + <value> <int>3</int> </value> + </item> + <item> + <key> <string>_bind_names</string> </key> + <value> + <object> + <klass> + <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/> + </klass> + <tuple/> + <state> + <dictionary> + <item> + <key> <string>_asgns</string> </key> + <value> + <dictionary> + <item> + <key> <string>name_container</string> </key> + <value> <string>container</string> </value> + </item> + <item> + <key> <string>name_context</string> </key> + <value> <string>context</string> </value> + </item> + <item> + <key> <string>name_m_self</string> </key> + <value> <string>script</string> </value> + </item> + <item> + <key> <string>name_subpath</string> </key> + <value> <string>traverse_subpath</string> </value> + </item> + </dictionary> + </value> + </item> + </dictionary> + </state> + </object> + </value> + </item> + <item> + <key> <string>_params</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>id</string> </key> + <value> <string>DataStreamModule_getTotalSize</string> </value> + </item> + </dictionary> + </pickle> + </record> +</ZopeData> diff --git a/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.py b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.py new file mode 100644 index 0000000000000000000000000000000000000000..ca0c0dcc4c5073f277a80337ab1d10f5244c1302 --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.py @@ -0,0 +1,18 @@ +portal = context.getPortalObject() +request = portal.REQUEST +reference = request['reference'] +data_chunk = request['data_chunk'] + +module = portal.data_stream_module +try: + data_stream = module[reference] +except KeyError: + data_stream = module.newContent(reference, 'Data Stream') + +append_method = context.getId() +if append_method == 'scalability_test_raw_lf': + data_stream.appendData(data_chunk + '\n') +else: + assert append_method == 'scalability_test_unpack', append_method + for time, data_chunk in context.unpack(data_chunk): + data_stream.appendData(data_chunk) diff --git a/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.xml b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.xml new file mode 100644 index 0000000000000000000000000000000000000000..967777861905d99ab61bd6e2bf75d54d9fdc5309 --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.xml @@ -0,0 +1,62 @@ +<?xml version="1.0"?> +<ZopeData> + <record id="1" aka="AAAAAAAAAAE="> + <pickle> + <global name="PythonScript" module="Products.PythonScripts.PythonScript"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>Script_magic</string> </key> + <value> <int>3</int> </value> + </item> + <item> + <key> <string>_bind_names</string> </key> + <value> + <object> + <klass> + <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/> + </klass> + <tuple/> + <state> + <dictionary> + <item> + <key> <string>_asgns</string> </key> + <value> + <dictionary> + <item> + <key> <string>name_container</string> </key> + <value> <string>container</string> </value> + </item> + <item> + <key> <string>name_context</string> </key> + <value> <string>context</string> </value> + </item> + <item> + <key> <string>name_m_self</string> </key> + <value> <string>script</string> </value> + </item> + <item> + <key> <string>name_subpath</string> </key> + <value> <string>traverse_subpath</string> </value> + </item> + </dictionary> + </value> + </item> + </dictionary> + </state> + </object> + </value> + </item> + <item> + <key> <string>_params</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>id</string> </key> + <value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value> + </item> + </dictionary> + </pickle> + </record> +</ZopeData> diff --git a/software/wendelin-scalability/test_scalability_fluentd/bt/dependency_list b/software/wendelin-scalability/test_scalability_fluentd/bt/dependency_list new file mode 100644 index 0000000000000000000000000000000000000000..b13b302d37641154d2b8ebdfdce6a683c2588f41 --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/bt/dependency_list @@ -0,0 +1 @@ +erp5_wendelin \ No newline at end of file diff --git a/software/wendelin-scalability/test_scalability_fluentd/bt/template_format_version b/software/wendelin-scalability/test_scalability_fluentd/bt/template_format_version new file mode 100644 index 0000000000000000000000000000000000000000..56a6051ca2b02b04ef92d5150c9ef600403cb1de --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/bt/template_format_version @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/software/wendelin-scalability/test_scalability_fluentd/bt/template_keep_last_workflow_history_only_path_list b/software/wendelin-scalability/test_scalability_fluentd/bt/template_keep_last_workflow_history_only_path_list new file mode 100644 index 0000000000000000000000000000000000000000..2424617ba957b5d9e87cdb478173730a7365d2e8 --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/bt/template_keep_last_workflow_history_only_path_list @@ -0,0 +1 @@ +portal_ingestion_policies/scalability_test_* \ No newline at end of file diff --git a/software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list b/software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list new file mode 100644 index 0000000000000000000000000000000000000000..92efc047751f0800a14fc8bcc80a39192a05e42d --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list @@ -0,0 +1,3 @@ +portal_ingestion_policies/scalability_test_* +portal_skins/custom/DataStreamModule_getTotalSize +portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion \ No newline at end of file diff --git a/software/wendelin-scalability/test_scalability_fluentd/bt/title b/software/wendelin-scalability/test_scalability_fluentd/bt/title new file mode 100644 index 0000000000000000000000000000000000000000..e8112a8a5dc919977c22bfccf0343f023e6effc9 --- /dev/null +++ b/software/wendelin-scalability/test_scalability_fluentd/bt/title @@ -0,0 +1 @@ +test_scalability_fluentd \ No newline at end of file diff --git a/stack/caddy/Caddyfile.in b/stack/caddy/Caddyfile.in index cb1c2ac79f17c969b19c25f5c326fa41bc2ede7f..bee9fc46718ccbdf936b40c559a52516e6138e17 100644 --- a/stack/caddy/Caddyfile.in +++ b/stack/caddy/Caddyfile.in @@ -1,5 +1,51 @@ -[$${caddy-configuration:ipv6}]:$${caddy-configuration:port} +{%- if parameter_dict['domain'] -%} -log $${caddy-configuration:access_log} +{{ parameter_dict['domain'] }} { -root $${directory:public_html} \ No newline at end of file +tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} { + + ca {{ca_custom_frontend_dict['rendered']}} + +} + +log {{caddy_configuration_dict['access_log']}} + +root {{ directory_dict['public_html'] }} + +{% if parameter_dict['enable-basic-auth'] == 'true' -%} +basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}} +{%- endif %} + +} + +{%- endif %} + +[{{caddy_configuration_dict['ipv6']}}]:{{parameter_dict['port-ipv6']}} { + + tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} { + + ca {{ certificate_authority_dict['ca-cert-file'] }} + + } + + log {{caddy_configuration_dict['access_log']}} + + root {{ directory_dict['public_html'] }} + + {% if parameter_dict['enable-basic-auth'] == 'true' -%} + basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}} + {%- endif %} + +} + +[{{caddy_configuration_dict['local_ip']}}]:{{parameter_dict['port-ipv4']}} { + + log {{caddy_configuration_dict['access_log']}} + + root {{ directory_dict['public_html'] }} + + {% if parameter_dict['enable-basic-auth'] == 'true' -%} + basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}} + {%- endif %} + +} \ No newline at end of file diff --git a/stack/caddy/buildout.cfg b/stack/caddy/buildout.cfg index 5a494ac88d5444ddd3646ae0baf31aad50f1ea2f..26f8e85a944c0b3f8a4186eecb228b40bae8e683 100644 --- a/stack/caddy/buildout.cfg +++ b/stack/caddy/buildout.cfg @@ -2,6 +2,7 @@ extends = ../../component/golang/buildout.cfg ../../stack/slapos.cfg + ../../component/dash/buildout.cfg gowork.cfg parts = @@ -14,36 +15,44 @@ parts = [gowork] install = github.com/mholt/caddy - + [instance-profile] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg.in -md5sum = ae83c35995ce14432ded78ca30cab61b +md5sum = 403f86b667f7a5d397993735bcd162ab output =${buildout:directory}/instance.cfg filename = instance.cfg mode = 0644 [template-caddyfile] -recipe = slapos.recipe.template +recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/Caddyfile.in -md5sum = 616f9c7cb788e1f1d7cee2093a7dc6ef -output = ${buildout:directory}/Caddyfile.in +#md5sum = bb3b314ebeb58e9875d547a053a9f268 +filename = Caddyfile.in +location = ${buildout:parts-directory}/${:_buildout_section_name_} mode = 0644 [template-caddy] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-caddy.cfg.in -md5sum = 045022e5698badca5dbdd6b95518f8e1 +md5sum = 3b796dee5c509393fd396fd019914a2e output = ${buildout:directory}/instance-caddy.cfg.in mode = 0644 [template-public-html] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/index.html +url = ${:_profile_base_location_}/templates/index.html md5sum = b5794ac8b10ed90173ad566e6e324b35 output = ${buildout:directory}/index.html mode = 0644 +[template-caddy-service] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/template-caddy-service.sh.in +md5sum = 5467fac7b95acde681e613ae98ce990d +output = ${buildout:directory}/template-caddy-service.sh.in +mode = 0644 + [caddy] recipe = slapos.recipe.cmmi path = ${go_github.com_mholt_caddy:location} diff --git a/stack/caddy/instance-caddy.cfg.in b/stack/caddy/instance-caddy.cfg.in index c1a8d54fe59064f8fa19cdc049913138f005fb54..85b5f1da7c8320fe34e96ff3193bbb9cee690117 100644 --- a/stack/caddy/instance-caddy.cfg.in +++ b/stack/caddy/instance-caddy.cfg.in @@ -2,10 +2,12 @@ parts = caddy-service caddy-configuration + certificate-authority + custom-cert + htpasswd public-html publish-connection-information - eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true @@ -20,29 +22,89 @@ service = $${:etc}/service public_html = $${buildout:directory}/public_html run = $${:var}/run log = $${:var}/log - -#www = $${:srv}/www -#ssl = $${:etc}/ssl +ca-dir = $${:srv}/ssl ################################# # caddy service ################################# [caddy-service] -recipe = slapos.cookbook:wrapper -command-line = ${caddy:output} -conf $${caddy-configuration:output} -wrapper-path = $${directory:service}/caddy -output = $${:wrapper-path} +recipe = slapos.recipe.template:jinja2 +template = ${template-caddy-service:output} +rendered = $${directory:service}/caddy +mode = 0700 +context = + key caddy_exec caddy-exec-dict:caddy-exec-file + section caddy_configuration_dict caddy-configuration + section parameter_dict slap-parameter + +[caddy-exec-dict] +caddy-exec-file = ${caddy:output} [caddy-configuration] -recipe = slapos.recipe.template -url = ${template-caddyfile:output} -output = $${directory:etc}/Caddyfile +recipe = slapos.recipe.template:jinja2 +template = ${template-caddyfile:location}/${template-caddyfile:filename} +rendered = $${directory:etc}/Caddyfile mode = 0600 access_log = $${directory:log}/caddy-access.log error_log = $${directory:log}/caddy-error.log ipv6 = $${slap-network-information:global-ipv6} local_ip = $${slap-network-information:local-ipv4} -port = 9443 +context = + section parameter_dict slap-parameter + section directory_dict directory + section caddy_configuration_dict caddy-configuration + section certificate_authority_dict certificate-authority + key htpasswd_dict htpasswd:passwd + section custom_cert_dict custom-cert + section ca_custom_frontend_dict ca-custom-frontend + +[ca-directory] +recipe = slapos.cookbook:mkdirectory +requests = $${directory:ca-dir}/requests/ +private = $${directory:ca-dir}/private/ +certs = $${directory:ca-dir}/certs +newcerts = $${directory:ca-dir}/newcerts/ +crl = $${directory:ca-dir}/crl/ + +[certificate-authority] +recipe = slapos.cookbook:certificate_authority +openssl-binary = ${openssl:location}/bin/openssl +ca-dir = $${directory:ca-dir} +requests-directory = $${ca-directory:requests} +wrapper = $${directory:service}/certificate_authority +ca-private = $${ca-directory:private} +ca-certs = $${ca-directory:certs} +ca-newcerts = $${ca-directory:newcerts} +ca-crl = $${ca-directory:crl} +ca-cert-file = $${:ca-dir}/cacert.pem +ca-key-file = $${:ca-private}/cakey.pem + +[custom-cert] +<= certificate-authority +recipe = slapos.cookbook:certificate_authority.request +executable = $${directory:service}/caddy +wrapper = $${directory:service}/caddy +key-file = $${ca-directory:private}/custom.key +cert-file = $${ca-directory:certs}/custom.crt +key-content = $${slap-parameter:key-content} +cert-content = $${slap-parameter:cert-content} + +[ca-custom-frontend] +recipe = slapos.recipe.template:jinja2 +template = $${template-empty:target} +rendered = $${ca-directory:certs}/caddy_frontend.ca.crt +context = + key content slap-parameter:caddy-ca-certificate + +[template-empty] +recipe = slapos.recipe.build:download +url = ${:_profile_base_location_}/templates/$${:filename} +filename = empty.in + +[htpasswd] +recipe = slapos.cookbook:generate.password +storage-path = $${directory:etc}/.pwd +bytes = 8 [public-html] recipe = slapos.recipe.template @@ -52,4 +114,18 @@ mode = 0600 [publish-connection-information] recipe = slapos.cookbook:publish -url = http://[$${caddy-configuration:ipv6}]:$${caddy-configuration:port} \ No newline at end of file +password = $${slap-parameter:password} +user = $${slap-parameter:username} +secure_access = https://[$${caddy-configuration:ipv6}]:$${slap-parameter:port-ipv6} + +[slap-parameter] +domain = +key-content = +cert-content = +caddy-ca-certificate = +port-ipv6 = 9443 +port-ipv4 = 4443 +enable-quic = true +enable-basic-auth = +username = admin +password = $${htpasswd:passwd} \ No newline at end of file diff --git a/stack/caddy/instance.cfg.in b/stack/caddy/instance.cfg.in index 36d0d0832987bb28b1bf06f138f8f23d6758b855..e585ca637440bb765341d2b3b6a1c21a0d6d5fac 100644 --- a/stack/caddy/instance.cfg.in +++ b/stack/caddy/instance.cfg.in @@ -6,7 +6,7 @@ [buildout] parts = switch-softwaretype - + # publish-connection-parameter # Define egg directories to be the one from Software Release @@ -28,7 +28,7 @@ rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename} filename = instance-caddy.cfg [slap-connection] -computer-id = {slap_connection:computer_id} +computer-id = $${slap_connection:computer_id} partition-id = $${slap_connection:partition_id} server-url = $${slap_connection:server_url} software-release-url = $${slap_connection:software_release_url} @@ -48,4 +48,4 @@ computer = $${slap_connection:computer_id} partition = $${slap_connection:partition_id} url = $${slap_connection:server_url} key = $${slap_connection:key_file} -cert = $${slap_connection:cert_file} +cert = $${slap_connection:cert_file} \ No newline at end of file diff --git a/stack/caddy/template-caddy-service.sh.in b/stack/caddy/template-caddy-service.sh.in new file mode 100644 index 0000000000000000000000000000000000000000..4d572e87f80400df3d25c92cdd9dd728dac738b7 --- /dev/null +++ b/stack/caddy/template-caddy-service.sh.in @@ -0,0 +1,9 @@ +#!${dash-output:dash} +# BEWARE: This file is operated by slapgrid +# BEWARE: It will be overwritten automatically + +{{ caddy_exec }} -conf {{caddy_configuration_dict['rendered']}} + {%- if parameter_dict['enable-quic'] != 'false' %} + -quic + {%- endif -%} + \ No newline at end of file diff --git a/stack/caddy/templates/empty.in b/stack/caddy/templates/empty.in new file mode 100644 index 0000000000000000000000000000000000000000..e02a27c0d8d22c4863637fdf92234b956b184df9 --- /dev/null +++ b/stack/caddy/templates/empty.in @@ -0,0 +1 @@ +{{ content }} \ No newline at end of file diff --git a/stack/caddy/index.html b/stack/caddy/templates/index.html similarity index 100% rename from stack/caddy/index.html rename to stack/caddy/templates/index.html diff --git a/stack/caucase/buildout.cfg b/stack/caucase/buildout.cfg index 42c018adb2ab78cfb126ac503372bcce9bfae2a3..025445099c61ee89e4005a3b10eec1beb793b48e 100644 --- a/stack/caucase/buildout.cfg +++ b/stack/caucase/buildout.cfg @@ -87,7 +87,7 @@ context = [versions] -Flask-User = 0.6.11 +Flask-User = 0.6.19 apache-libcloud = 2.1.0 bcrypt = 3.1.3 caucase = 0.1.4 @@ -103,24 +103,24 @@ smmap2 = 2.0.3 Flask-AlchemyDumps = 0.0.10 # Required by: -# Flask-User==0.6.11 +# Flask-User==0.6.19 Flask-Login = 0.4.0 # Required by: -# Flask-User==0.6.11 +# Flask-User==0.6.19 Flask-Mail = 0.9.1 # Required by: # Flask-AlchemyDumps==0.0.10 -# Flask-User==0.6.11 -Flask-SQLAlchemy = 2.2 +# Flask-User==0.6.19 +Flask-SQLAlchemy = 2.3.2 # Required by: # Flask-AlchemyDumps==0.0.10 -Flask-Script = 2.0.5 +Flask-Script = 2.0.6 # Required by: -# Flask-User==0.6.11 +# Flask-User==0.6.19 Flask-WTF = 0.14.2 # Required by: @@ -133,7 +133,7 @@ PyRSS2Gen = 1.1 # Required by: # Flask-AlchemyDumps==0.0.10 -SQLAlchemy = 1.1.11 +SQLAlchemy = 1.1.15 # Required by: # Flask-AlchemyDumps==0.0.10 @@ -173,8 +173,9 @@ passlib = 1.7.1 # Required by: # caucase==0.1.4 -pem = 16.1.0 +pem = 17.1.0 +# Required by: # caucase==0.1.4 pyasn1 = 0.2.3 @@ -183,5 +184,5 @@ pyasn1 = 0.2.3 pyasn1-modules = 0.0.9 # Required by: -# Flask-User==0.6.11 -pycryptodome = 3.4.6 +# Flask-User==0.6.19 +pycryptodome = 3.4.7 diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index 45d41f8216595cac19094e2490bbc14c6bb90bc3..c19a927e2a76287b46b593bc404e5438420ed912 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -476,7 +476,6 @@ eggs = ${neoppod:eggs} collective.recipe.template coverage erp5diff - inotifyx interval ipdb Jinja2 diff --git a/stack/erp5/buildout.hash.cfg b/stack/erp5/buildout.hash.cfg index 19413749400dc2580c56c27866e5ad002beeb79a..c60d2eecfc8039d2c8b1d1b4cf2dc09143a956e7 100644 --- a/stack/erp5/buildout.hash.cfg +++ b/stack/erp5/buildout.hash.cfg @@ -39,7 +39,7 @@ md5sum = 76f9e8c8cdc352081e34539d8fc17026 [template-zope-conf] filename = zope.conf.in -md5sum = bbea91bc8f0b2d455d9824928abfad5b +md5sum = 3524ef2e14cea4a5bd40fdc9e95cfc0c [site-zcml] filename = site.zcml @@ -51,7 +51,7 @@ md5sum = d814b984abf2dc444af2a0bc6287e7f5 [template-mariadb-initial-setup] filename = mariadb_initial_setup.sql.in -md5sum = 6465212fdc7fe9076a0c929d9f14da14 +md5sum = dec33a617fa1b307c8ddb883efcfe3ce [template-postfix] filename = instance-postfix.cfg.in @@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e [template-erp5] filename = instance-erp5.cfg.in -md5sum = 33c8f97afe58058d428108fb933523ac +md5sum = 2465af81147af322056cee9f6c7de14f [template-zeo] filename = instance-zeo.cfg.in @@ -87,7 +87,7 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba [template-zope] filename = instance-zope.cfg.in -md5sum = 27d26c6380883cf3bd7b2f003f7888d8 +md5sum = fd7e8c507cef1950e6c0347ce2a01021 [template-balancer] filename = instance-balancer.cfg.in diff --git a/stack/erp5/instance-erp5.cfg.in b/stack/erp5/instance-erp5.cfg.in index a31696bf351551f151c9a41d606b14fec070b13d..2afebc4389a7e7db1842599ebe54cb16b090c554 100644 --- a/stack/erp5/instance-erp5.cfg.in +++ b/stack/erp5/instance-erp5.cfg.in @@ -155,6 +155,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }} config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }} +config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }} config-inituser-login = {{ dumps(inituser_login) }} config-inituser-password = ${publish-early:inituser-password} config-kumofs-url = ${request-memcached-persistent:connection-url} diff --git a/stack/erp5/instance-zope.cfg.in b/stack/erp5/instance-zope.cfg.in index 68f5cbed05ccb2dbb88f6af2f9bcaa49e34a0182..0f3f7c66d82477340216304b3062f86cbd5cf351 100644 --- a/stack/erp5/instance-zope.cfg.in +++ b/stack/erp5/instance-zope.cfg.in @@ -230,6 +230,7 @@ sql-connection-string = {{ '%s@erp5-catalog-0:%s %s %s' % ( mysql.path.split('/')[1], mysql.port, mysql.username, mysql.password) }} bt5 = {{ slapparameter_dict['bt5'] }} bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }} +id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }} home = ${buildout:directory} # We only want to change the hostname to 'erp5-cloudooo' if we use the internal # cloudooo. We plan to remove the ability to have an internal one, so this diff --git a/stack/erp5/mariadb_initial_setup.sql.in b/stack/erp5/mariadb_initial_setup.sql.in index c43a4c045bd41f740cfb66a8a982f4ee92876837..49fafb854fc56f39d82f34b5a6b86ee6c36dadde 100644 --- a/stack/erp5/mariadb_initial_setup.sql.in +++ b/stack/erp5/mariadb_initial_setup.sql.in @@ -1,7 +1,11 @@ USE mysql; DROP FUNCTION IF EXISTS last_insert_grn_id; +DROP FUNCTION IF EXISTS mroonga_snippet; +DROP FUNCTION IF EXISTS mroonga_command; {% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%} -{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';{% endif %} +{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}'; +CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}'; +CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %} DROP FUNCTION IF EXISTS sphinx_snippets; #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so'; diff --git a/stack/erp5/zope.conf.in b/stack/erp5/zope.conf.in index 2d7590a23b0724e7caa3d06b679b9bde64037d88..171a688cfeceb68dff0cc4f6195c94f2021d422f 100644 --- a/stack/erp5/zope.conf.in +++ b/stack/erp5/zope.conf.in @@ -86,6 +86,9 @@ products {{ parameter_dict['instance-products'] }} cmf_activity_sql_connection_string {{ sql_connection_string }} bt5_repository_url {{ ' '.join(bt5_repository_url) }} bt5 {{ parameter_dict['bt5'] }} +{%- if parameter_dict['id-store-interval'] != None %} + id_store_interval {{ parameter_dict['id-store-interval'] }} +{%- endif %} cloudooo_url {{ parameter_dict['cloudooo-url'] }} </product-config> diff --git a/stack/slapos.cfg b/stack/slapos.cfg index b45325e697800c41689f416ab67b8ea492ee4681..9452ffb45eabd89e845b7fc70bc38d7c416217c1 100644 --- a/stack/slapos.cfg +++ b/stack/slapos.cfg @@ -78,8 +78,6 @@ recipe = zc.recipe.egg:develop setup = ${slapos.cookbook-repository:location} # Install slapos.cookbook containing all officials recipes -# Explicitely define dependencies as well, because of buildout limitation -# if using "develop". XXX really ? [slapos-cookbook] recipe = zc.recipe.egg eggs = @@ -91,15 +89,6 @@ eggs = pyOpenSSL slapos.cookbook slapos.libnetworkcache - hexagonit.recipe.download - inotifyx - netaddr - netifaces - requests - slapos.core - supervisor - xml_marshaller - pytz [versions] # Use SlapOS patched zc.buildout @@ -139,7 +128,7 @@ requests = 2.13.0 setuptools = 33.1.1 six = 1.10.0 slapos.cookbook = 1.0.53 -slapos.core = 1.4.2 +slapos.core = 1.4.3 slapos.extension.strip = 0.4 slapos.libnetworkcache = 0.15 slapos.rebootstrap = 4.1 @@ -151,7 +140,7 @@ xml-marshaller = 0.9.7 paramiko = 2.1.3 # Required by: -# slapos.core==1.4.2 +# slapos.core==1.4.3 Flask = 0.12 # Required by: @@ -171,7 +160,7 @@ ipaddress = 1.0.18 jsonschema = 2.6.0 # Required by: -# slapos.core==1.4.2 +# slapos.core==1.4.3 # XXX 'slapos node format' raises an exception with netifaces 0.10.5. netifaces = 0.10.4 @@ -184,15 +173,15 @@ packaging = 16.8 pycparser = 2.17 # Required by: -# slapos.core==1.4.2 +# slapos.core==1.4.3 supervisor = 3.3.3 # Required by: -# slapos.core==1.4.2 +# slapos.core==1.4.3 uritemplate = 3.0.0 # Required by: -# slapos.core==1.4.2 +# slapos.core==1.4.3 zope.interface = 4.3.3 [networkcache] @@ -200,7 +189,6 @@ download-cache-url = http://download.shacache.org/ download-dir-url = http://dir.shacache.org/ # signature certificates of the following uploaders. -# Romain Courteaud # Sebastien Robin # Kazuhiko Shiozaki # Gabriel Monnerat @@ -210,19 +198,6 @@ download-dir-url = http://dir.shacache.org/ # Ivan Tyagov # Julien Muchembled signature-certificate-list = - -----BEGIN CERTIFICATE----- - MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE - CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5 - MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl - ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF - AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw - boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX - Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA - ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX - mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC - q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g - QUUGLQ== - -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw