Commit 58f2c3a6 authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 009513b1 9437ec3a
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
extends = extends =
../coreutils/buildout.cfg ../coreutils/buildout.cfg
../patch/buildout.cfg ../patch/buildout.cfg
../randomsleep/buildout.cfg
parts = dcron-output parts = dcron-output
...@@ -18,6 +19,7 @@ make-options = ...@@ -18,6 +19,7 @@ make-options =
post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook
environment = environment =
PATH=${patch:location}/bin:%(PATH)s PATH=${patch:location}/bin:%(PATH)s
randomsleep_install = ${randomsleep:location}
[dcron-output] [dcron-output]
# Shared binary location to ease migration # Shared binary location to ease migration
......
...@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location} ...@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location}
[golang18] [golang18]
<= golang-common <= golang-common
url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz url = https://dl.google.com/go/go1.8.7.src.tar.gz
md5sum = 64e9380e07bba907e26a00cf5fcbe77e md5sum = c61cfe9c85e7d42f903d3fe146d7cde6
# go1.8 needs go1.4 to bootstrap # go1.8 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -43,8 +43,8 @@ environment-extra = ...@@ -43,8 +43,8 @@ environment-extra =
[golang19] [golang19]
<= golang-common <= golang-common
url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz url = https://dl.google.com/go/go1.9.4.src.tar.gz
md5sum = 44105c865a1a810464df79233a05a568 md5sum = 6816441fd6680c63865cdd5cb8bc1960
# go1.9 needs go1.4 to bootstrap # go1.9 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -63,6 +63,11 @@ environment-extra = ...@@ -63,6 +63,11 @@ environment-extra =
# lab.nexedi.com/kirr/neo/go/... \ # lab.nexedi.com/kirr/neo/go/... \
# github.com/pkg/profile \ # github.com/pkg/profile \
# golang.org/x/perf/cmd/benchstat # golang.org/x/perf/cmd/benchstat
#
# it is possible to specify Go build flags used for compilation e.g. this way:
#
# [gowork]
# buildflags = -race
[gowork] [gowork]
directory = ${buildout:directory}/go.work directory = ${buildout:directory}/go.work
src = ${:directory}/src src = ${:directory}/src
...@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe} ...@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications) # go version used for the workspace (possible to override in applications)
golang = ${golang19:location} golang = ${golang19:location}
# no special build flags by default
buildflags =
# everything is done by dependent parts # everything is done by dependent parts
recipe = plone.recipe.command recipe = plone.recipe.command
command = : command = :
...@@ -100,7 +108,7 @@ stop-on-error = true ...@@ -100,7 +108,7 @@ stop-on-error = true
# clients should put package list to install to gowork:install ("..." requests installing everything) # clients should put package list to install to gowork:install ("..." requests installing everything)
[gowork.goinstall] [gowork.goinstall]
recipe = plone.recipe.command recipe = plone.recipe.command
command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}" command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v ${gowork:install}"
update-command = ${:command} update-command = ${:command}
stop-on-error = true stop-on-error = true
...@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git ...@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git
# 3) provide repository (which is not the same as importpath in general case) # 3) provide repository (which is not the same as importpath in general case)
# #
# the list of go packages for a go workspace state can be automatically # the list of go packages for a go workspace state can be automatically
# generated with the help of go-pkg-snapshot tool. # generated with the help of gowork-snapshot tool.
[go-git-package] [go-git-package]
<= git-repository <= git-repository
location = ${gowork:src}/${:go.importpath} location = ${gowork:src}/${:go.importpath}
...@@ -27,8 +27,8 @@ parts = ...@@ -27,8 +27,8 @@ parts =
[mariadb] [mariadb]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.1.30 version = 10.1.31
md5sum = c424fd12bdff388e3da1bdecf42626c9 md5sum = 14ab0398c019eb531bc29f2c437ccb51
patch-options = -p0 patch-options = -p0
patches = patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3 ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......
From efa86aa5ad5b89ec329b3cde288cb119efb21c89 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Thu, 06 Jul 2017 21:15:31 +0200
Subject: [PATCH] asyncore: switch default to use poll instead of select
Recently we started to get the following errors on a Wendelin production
instance:
File ".../bin/runzope", line 211, in <module>
sys.exit(Zope2.Startup.run.run())
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/run.py", line 26, in run
starter.run()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/__init__.py", line 111, in run
Lifetime.loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 43, in loop
lifetime_loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 53, in lifetime_loop
asyncore.poll(timeout, map)
File ".../parts/python2.7/lib/python2.7/asyncore.py", line 145, in poll
r, w, e = select.select(r, w, e, timeout)
ValueError: filedescriptor out of range in select()
Initially we thought the reason here is that number of file descriptors this
process uses goes beyond allowed limit (65K open files on that instance)
because wendelin.core uses 1 fd per an array view (opening
/dev/shm/ramh.XXXXX) but that turned out to be not strictly that case:
The reason here is that select() has limit for how many #fd it can
monitor at all. The limit is system-specific but on Linux it is usually
1024 - http://man7.org/linux/man-pages/man2/select.2.html#BUGS :
$ cat s.c
#include <sys/select.h>
#include <stdio.h>
int main() {
printf("%d\n", FD_SETSIZE);
return 0;
}
$ tcc -run s.c
1024
Also select() can monitor only file descriptors which are by itself
"< FD_SETSIZE", e.g. it cannot monitor fd=1025 even if there are only 10
opened file descriptors because 1025 is not < 1024.
As was said above in wendelin.core every array view uses 1 file
descriptor, so if we are using not so small amount of arrays, even
though #fd does not go beyond process-specific ulimit, because of
select() usage the total number of allowed opened file descriptors
becomes essentially 1024.
So let's switch from select() to poll() which does not have this 1024
#fd limit. Asyncore already support using poll() out of the box -
either via passing use_pull=True to asyncore.loop()
https://docs.python.org/2/library/asyncore.html#asyncore.loop
or by using asyncore.poll2() instead of asyncore.poll()
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L170
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L125
--------
One option would be to switch asyncore.poll() -> asyncore.poll2() for only 1
place in Zope - inside lifetime_loop(). There are however many such similar
places in Zope (e.g. in medusa) and in other software.
For this reason, for me, what makes sense to do is not to patch all such
places, but instead change via runtime-patching asyncore.poll to be
asyncore.poll2 - this way all software will automatically benefit from
poll() usage instead of select.
P.S.
What might also make sense to do in the future is to actually let
asyncore.poll to use epoll(), as both select() and poll() are doing all
fd registration on _every_ call, so when #fd grows this preparation
time grows too. For epoll() file descriptors are registered only once.
For this to work asyncore.socket_map has to be patched also, since there
are places in code which modify this fd registry directly (e.g. remove
fd from there etc)
Original patch & discussion: https://lab.nexedi.com/kirr/Zope/commit/c6addb05
---
Lib/asyncore.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 105982f..a3025a4 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -661,3 +661,12 @@ if os.name == 'posix':
self.socket = file_wrapper(fd)
self._fileno = self.socket.fileno()
self.add_channel()
+
+# monkeypatch to switch everyone's default to use poll instead of select
+# (select has 1024 fd limit)
+# TODO it is better to use epoll
+_poll_select = poll
+_poll_poll = poll2
+poll = _poll_poll
+
+#print >>sys.stderr, 'I(nxd): asyncore: using poll instead of select'
--
2.16.1.101.gde0f0111ea
...@@ -43,6 +43,7 @@ patches = ...@@ -43,6 +43,7 @@ patches =
${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7 ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a ${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a
${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e ${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
url = url =
http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
configure-options = configure-options =
......
...@@ -19,12 +19,11 @@ extends = ...@@ -19,12 +19,11 @@ extends =
[kvm] [kvm]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
# qemu-kvm and qemu are now the same since 1.3. # qemu-kvm and qemu are now the same since 1.3.
url = http://wiki.qemu-project.org/download/qemu-2.8.0.tar.bz2 url = http://wiki.qemu-project.org/download/qemu-2.11.0.tar.bz2
md5sum = 17940dce063b6ce450a12e719a6c9c43 md5sum = 335994a755bc655e88a87aeb36bfc0b9
configure-options = configure-options =
--target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu" --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
--enable-system --enable-system
--with-system-pixman
--disable-sdl --disable-sdl
--disable-xen --disable-xen
--disable-vnc-sasl --disable-vnc-sasl
...@@ -58,22 +57,18 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -58,22 +57,18 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
<= debian-netinst-base <= debian-netinst-base
arch = amd64 arch = amd64
[debian-amd64-squeeze-netinst.iso] [debian-amd64-jessie-netinst.iso]
<= debian-amd64-netinst-base # Download the installer of Debian 8 (Jessie)
version = 6.0.10
md5sum = 7f82d341561035f65933da43f94d5b52
[debian-amd64-wheezy-netinst.iso]
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
version = 7.11.0 release = archive
md5sum = 096c1c18b44c269808bd815d58c53c8f version = 8.10.0
md5sum = 19dcfc381bd3e609c6056216d203f5bc
[debian-amd64-netinst.iso] [debian-amd64-netinst.iso]
# Download the installer of Debian 8 (Jessie)
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
release = release/current release = release/current
version = 8.6.0 version = 9.3.0
md5sum = e9f61bf327db6d8f7cee05a99f2353cc md5sum = db8ab7871bc2b7d456c4746e706fb5d3
[debian-amd64-testing-netinst.iso] [debian-amd64-testing-netinst.iso]
# Download the installer of Debian Stretch # Download the installer of Debian Stretch
......
[buildout]
extends =
../coreutils/buildout.cfg
../bash/buildout.cfg
[randomsleep]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin_dir = ${buildout:bin-directory}
bash_script_code =
if [ "$#" -ne 1 ]; then
echo "usage: randomsleep maxseconds"
exit
fi
${coreutils:location}/bin/sleep $((RANDOM % $1)).$((RANDOM % 100))
wrapper_script_code =
#!${bash:location}/bin/bash
${bash:location}/bin/bash ${:location}/randomsleep.bash "$@"
script =
os.makedirs(self.options['location'])
bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
with open(bash_script_path, 'w') as f:
f.write(self.options['bash_script_code'])
wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
with open(wrapper_script_path, 'w') as f:
f.write(self.options['wrapper_script_code'])
os.chmod(wrapper_script_path, 0750)
diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
index b386d59..f03b056 100755 index 88fb5a2..782dbc5 100755
--- a/tensorboard/pip_package/build_pip_package.sh --- a/tensorboard/pip_package/build_pip_package.sh
+++ b/tensorboard/pip_package/build_pip_package.sh +++ b/tensorboard/pip_package/build_pip_package.sh
@@ -26,6 +26,7 @@ function main() { @@ -26,6 +26,7 @@ function main() {
...@@ -10,14 +10,14 @@ index b386d59..f03b056 100755 ...@@ -10,14 +10,14 @@ index b386d59..f03b056 100755
echo $(date) : "=== Using tmpdir: ${TMPDIR}" echo $(date) : "=== Using tmpdir: ${TMPDIR}"
@@ -45,8 +46,8 @@ function main() { @@ -44,9 +45,7 @@ function main() {
pushd ${TMPDIR} >/dev/null
rm -f MANIFEST rm -f MANIFEST
echo $(date) : "=== Building wheel" echo $(date) : "=== Building python2 wheel in $PWD"
echo $(pwd) - python setup.py bdist_wheel --python-tag py2 >/dev/null
- python setup.py bdist_wheel >/dev/null - echo $(date) : "=== Building python3 wheel in $PWD"
- python3 setup.py bdist_wheel >/dev/null - python setup.py bdist_wheel --python-tag py3 >/dev/null
+ PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null + PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
+ #python3 setup.py bdist_wheel >/dev/null
mkdir -p ${DEST} mkdir -p ${DEST}
cp dist/* ${DEST} cp dist/* ${DEST}
popd popd >/dev/null
...@@ -11,11 +11,11 @@ parts = ...@@ -11,11 +11,11 @@ parts =
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
repository = https://github.com/tensorflow/tensorboard repository = https://github.com/tensorflow/tensorboard
tag = 0.4 tag = 0.4.0
git-binary = ${git:location}/bin/git git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1) command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
[tensorboard-build] [tensorboard-build]
recipe = slapos.recipe.build recipe = slapos.recipe.build
...@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter} ...@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script = script =
os.makedirs(location) os.makedirs(location)
workdir = self.options['tensorboard-repository-path'] workdir = self.options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg' egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist') dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name) dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'], env = {'PATH':':'.join([self.options['bazel-bin'],
......
...@@ -5,8 +5,8 @@ extends = ...@@ -5,8 +5,8 @@ extends =
[util-linux] [util-linux]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.kernel.org/pub/linux/utils/util-linux/v2.29/util-linux-2.29.tar.xz url = https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.1.tar.xz
md5sum = 07b6845f48a421ad5844aa9d58edb837 md5sum = 7733b583dcb51518944d42aa62ef19ea
configure-options = configure-options =
--disable-static --disable-static
--enable-libuuid --enable-libuuid
...@@ -29,7 +29,7 @@ configure-options = ...@@ -29,7 +29,7 @@ configure-options =
--disable-rename --disable-rename
--disable-schedutils --disable-schedutils
--disable-setterm --disable-setterm
--disable-libsmartcols --enable-libsmartcols
--disable-switch_root --disable-switch_root
--disable-tinfo --disable-tinfo
--disable-tls --disable-tls
......
[buildout]
parts=
zbar
[zbar]
recipe = slapos.recipe.cmmi
url = https://jaist.dl.sourceforge.net/project/zbar/zbar/0.10/zbar-0.10.tar.bz2
md5sum = 0fd61eb590ac1bab62a77913c8b086a5
configure-options =
--disable-video
--without-imagemagick
--without-gtk
--without-xv
--without-qt
--without-python
--without-x
--without-jpg
environment =
CFLAGS=
[buildout]
extends =
../pillow/buildout.cfg
../zbar/buildout.cfg
parts =
zbarlight
[zbarlight]
recipe = zc.recipe.egg:custom
egg = zbarlight
include-dirs =
${zbar:location}/include
library-dirs =
${zbar:location}/lib
rpath =
${zbar:location}/lib
depend-eggs = ${pillow-python:egg}
...@@ -50,8 +50,8 @@ gitdb = 0.6.4 ...@@ -50,8 +50,8 @@ gitdb = 0.6.4
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.download = 1.0 slapos.recipe.download = 1.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -10,8 +10,8 @@ gitdb = 0.6.4 ...@@ -10,8 +10,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
numpy = 1.11.2 numpy = 1.11.2
pyasn1 = 0.2.3 pyasn1 = 0.2.3
......
...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3 ...@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
erp5.util = 0.4.49 erp5.util = 0.4.49
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12 ...@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies # indirect dependancies
cp.recipe.cmd = 0.5 cp.recipe.cmd = 0.5
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
zope.exceptions = 4.0.7 zope.exceptions = 4.0.7
zope.testing = 4.1.3 zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0 zc.recipe.testrunner = 2.0.0
......
...@@ -117,7 +117,7 @@ ...@@ -117,7 +117,7 @@
"timerserver-interval": { "timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable", "description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"default": 5, "default": 5,
"type": "integer" "type": "number"
}, },
"ssl-authentication": { "ssl-authentication": {
"title": "Enable SSL Client authentication on this zope instance.", "title": "Enable SSL Client authentication on this zope instance.",
......
...@@ -59,7 +59,7 @@ mode = 0644 ...@@ -59,7 +59,7 @@ mode = 0644
[versions] [versions]
PyXML = 0.8.5 PyXML = 0.8.5
erp5.util = 0.4.50 erp5.util = 0.4.50
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
ipython = 5.3.0 ipython = 5.3.0
apache-libcloud = 2.1.0 apache-libcloud = 2.1.0
gitdb2 = 2.0.2 gitdb2 = 2.0.2
......
...@@ -106,6 +106,7 @@ def waitForSite(partition_path): ...@@ -106,6 +106,7 @@ def waitForSite(partition_path):
date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)), date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration=end - start, duration=end - start,
) )
print(try_info + 'status_dict %r' % (status_dict,))
return status_dict return status_dict
......
...@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf ...@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3 ...@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12 docutils = 0.12
plone.recipe.command = 1.1 plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001 rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
...@@ -72,7 +72,7 @@ async = 0.6.1 ...@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.40.4 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
plone.recipe.command = 1.1 plone.recipe.command = 1.1
......
...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8 ...@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -113,5 +113,5 @@ mode = 0644 ...@@ -113,5 +113,5 @@ mode = 0644
[versions] [versions]
erp5.util = 0.4.50 erp5.util = 0.4.50
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
selenium = 3.8.0 selenium = 3.8.0
...@@ -85,7 +85,7 @@ pyzmq = 16.0.2 ...@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1 scikit-learn = 0.18.1
seaborn = 0.7.1 seaborn = 0.7.1
simplegeneric = 0.8.1 simplegeneric = 0.8.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
statsmodels = 0.8.0 statsmodels = 0.8.0
terminado = 0.6 terminado = 0.6
tornado = 4.4.2 tornado = 4.4.2
......
...@@ -41,6 +41,7 @@ parts = ${:common-parts} ...@@ -41,6 +41,7 @@ parts = ${:common-parts}
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
interpreter = python.eggs
eggs = eggs =
${python-cffi:egg} ${python-cffi:egg}
${python-cryptography:egg} ${python-cryptography:egg}
...@@ -89,7 +90,7 @@ command = ...@@ -89,7 +90,7 @@ command =
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
md5sum = bf5ef731c0d8da0267a4939882b4eeee md5sum = 5a17fc127190bbc19361c5ffb10711b3
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
...@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download ...@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644 mode = 644
md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d md5sum = e3cc9ffe857da1078e321cab65173fb1
download-only = true download-only = true
on-update = true on-update = true
...@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download ...@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644 mode = 644
md5sum = 05b6004e8c7a94de14f247affcef4971 md5sum = 26d931a0279d49eadb3881f440b623bc
download-only = true download-only = true
on-update = true on-update = true
...@@ -185,7 +186,7 @@ ignore-existing = true ...@@ -185,7 +186,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/template-kvm-run.in url = ${:_profile_base_location_}/template/template-kvm-run.in
mode = 644 mode = 644
filename = template-kvm-run.in filename = template-kvm-run.in
md5sum = 178a24cdad77cb6c2e519ac629dd0e74 md5sum = 122bf5e8c7a12bceef1bdd6d6b54f4d7
download-only = true download-only = true
on-update = true on-update = true
...@@ -195,7 +196,7 @@ ignore-existing = true ...@@ -195,7 +196,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/kvm-controller-run.in url = ${:_profile_base_location_}/template/kvm-controller-run.in
mode = 644 mode = 644
filename = kvm-controller-run.in filename = kvm-controller-run.in
md5sum = 71afd2d13f6e56993ae413a168e012d7 md5sum = 7e6c79232cc88c15ed21c112ff801b76
download-only = true download-only = true
on-update = true on-update = true
...@@ -225,7 +226,7 @@ ignore-existing = true ...@@ -225,7 +226,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/qemu-is-ready.in url = ${:_profile_base_location_}/template/qemu-is-ready.in
mode = 644 mode = 644
filename = qemu-is-ready.in filename = qemu-is-ready.in
md5sum = 0066fa0f5f3dd47bded5e5924df2550d md5sum = b304eec8e2cb71f10ea83cac22f6db12
download-only = true download-only = true
on-update = true on-update = true
......
...@@ -189,13 +189,35 @@ ...@@ -189,13 +189,35 @@
"default": "started", "default": "started",
"enum": ["started", "stopped"] "enum": ["started", "stopped"]
}, },
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": { "ram-size": {
"title": "RAM size", "title": "RAM size",
"description": "RAM size, in MB.", "description": "RAM size, in MB.",
"type": "integer", "type": "integer",
"default": 1024, "default": 1024,
"minimum": 128, "minimum": 1024,
"multipleOf": 128 "multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
}, },
"auto-ballooning": { "auto-ballooning": {
"title": "Enable qemu auto ballooning.", "title": "Enable qemu auto ballooning.",
...@@ -245,10 +267,13 @@ ...@@ -245,10 +267,13 @@
"type": "integer", "type": "integer",
"minimum": 1 "minimum": 1
}, },
"cpu-options": { "cpu-max-count": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.", "title": "Maximum CPU amount",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.", "description": "Specifies the maximum number of CPUs.",
"type": "string" "type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
}, },
"numa": { "numa": {
"title": "Simulate a multi node NUMA system.", "title": "Simulate a multi node NUMA system.",
......
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
{% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%} {% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
{% set kvm_instance_dict = {} -%} {% set kvm_instance_dict = {} -%}
{% set kvm_hostname_list = [] -%} {% set kvm_hostname_list = [] -%}
{% set monitor_url_list = [] -%} {% set monitor_base_url_dict = {} -%}
{% macro setconfig(name, value) -%} {% macro setconfig(name, value) -%}
{# will set a config-name = value if value is not empty -#} {# will set a config-name = value if value is not empty -#}
...@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi ...@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }} config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }} config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }} config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', '50G')) }}
config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }} config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }} config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }} config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', 24)) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }} {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }} {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
{{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }} {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
{{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }} {{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}
...@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True ...@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
{{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }} {{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%} {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
{{ setconfig('nat-rules', ' '.join(nat_rules_list)) }} {{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
config-publish-nat-url = True config-publish-nat-url = True
config-use-nat = {{ use_nat }} config-use-nat = {{ use_nat }}
config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }} config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
...@@ -124,10 +127,10 @@ return = ...@@ -124,10 +127,10 @@ return =
{% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%} {% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%}
{{ ' ' }}tap-ipv4 {{ ' ' }}tap-ipv4
{% do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%} {% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%} {% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% endif -%} {% endif -%}
{% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%} {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%} {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
{% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%} {% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}
...@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of ...@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of
username = admin username = admin
password = ${monitor-htpasswd:passwd} password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters] [monitor-base-url-dict]
monitor-url-list += {% for key, value in monitor_base_url_dict.items() -%}
{% for url in monitor_url_list -%} {{ key }} = {{ value }}
{{ ' ' ~ url }}
{% endfor %} {% endfor %}
[monitor-conf-parameters]
private-path-list += private-path-list +=
${directory:webroot}/ ${directory:webroot}/
......
...@@ -4,13 +4,35 @@ ...@@ -4,13 +4,35 @@
"title": "Input Parameters", "title": "Input Parameters",
"properties": { "properties": {
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": { "ram-size": {
"title": "RAM size", "title": "RAM size",
"description": "RAM size, in MB.", "description": "RAM size, in MB.",
"type": "integer", "type": "integer",
"default": 1024, "default": 1024,
"minimum": 128, "minimum": 1024,
"multipleOf": 128 "multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
}, },
"auto-ballooning": { "auto-ballooning": {
"title": "Enable qemu auto ballooning.", "title": "Enable qemu auto ballooning.",
...@@ -54,10 +76,25 @@ ...@@ -54,10 +76,25 @@
"type": "integer", "type": "integer",
"minimum": 1 "minimum": 1
}, },
"cpu-options": { "cpu-max-count": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.", "title": "Maximum CPU amount",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.", "description": "Specifies the maximum number of CPUs.",
"type": "string" "type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
},
"max-cpu-hotplug-count": {
"title": "Maximum hotpluggable CPU amount",
"description": "Specifies the maximum number of hotpluggable CPUs.",
"type": "integer",
"default": 1
},
"cpu-hotplug-amount": {
"title": "CPU hotplug amount",
"description": "Specifies the number of CPUs to hotplug",
"type": "integer",
"default": 0
}, },
"numa": { "numa": {
"title": "Simulate a multi node NUMA system.", "title": "Simulate a multi node NUMA system.",
......
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%} {% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
{% set name = slapparameter_dict.get('name', 'localhost') -%} {% set name = slapparameter_dict.get('name', 'localhost') -%}
{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%} {% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%} {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set frontend_software_type = 'default' -%} {% set frontend_software_type = 'default' -%}
...@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd ...@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd
bytes = 8 bytes = 8
[kvm-controller-parameter-dict] [kvm-controller-parameter-dict]
python-path = {{ python_executable }} python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd} vnc-passwd = ${gen-passwd:passwd}
socket-path = ${directory:var}/qmp_socket socket-path = ${directory:var}/qmp_socket
pid-file = ${directory:run}/pid_file
kvm-status-path = ${directory:var}/qemu-vm-is-ready kvm-status-path = ${directory:var}/qemu-vm-is-ready
cpu-count = ${slap-parameter:cpu-count}
ram-hotplug-slot-size = ${slap-parameter:ram-hotplug-slot-size}
ram-size = ${slap-parameter:ram-size}
enable-device-hotplug = ${slap-parameter:enable-device-hotplug}
[kvm-parameter-dict] [kvm-parameter-dict]
python-path = {{ python_executable }} python-path = {{ python_executable }}
...@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size} ...@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size}
disk-type = ${slap-parameter:disk-type} disk-type = ${slap-parameter:disk-type}
pid-file-path = ${directory:run}/pid_file pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path} socket-path = ${kvm-controller-parameter-dict:socket-path}
smp-count = ${slap-parameter:cpu-count} enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-options = ${slap-parameter:cpu-options} smp-count = ${kvm-controller-parameter-dict:cpu-count}
ram-size = ${slap-parameter:ram-size} smp-max-count = ${slap-parameter:cpu-max-count}
numa = ${slap-parameter:numa}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = ${slap-parameter:ram-max-size}
init-ram-size = 1024
mac-address = ${create-mac:mac-address} mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address} tap-mac-address = ${create-tap-mac:mac-address}
...@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache} ...@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache}
disk-aio = ${slap-parameter:disk-aio} disk-aio = ${slap-parameter:disk-aio}
auto-ballooning = ${slap-parameter:auto-ballooning} auto-ballooning = ${slap-parameter:auto-ballooning}
machine-options = ${slap-parameter:machine-options} machine-options = ${slap-parameter:machine-options}
cpu-options = ${slap-parameter:cpu-model} cpu-hotplug-slot-size = ${slap-parameter:cpu-model}
log-file = ${directory:log}/qemu.log log-file = ${directory:log}/qemu.log
...@@ -550,15 +559,15 @@ nbd-host = ...@@ -550,15 +559,15 @@ nbd-host =
nbd2-port = 1024 nbd2-port = 1024
nbd2-host = nbd2-host =
enable-device-hotplug = False
ram-size = 1024 ram-size = 1024
ram-max-size = 50G
ram-hotplug-slot-size = 512
disk-size = 10 disk-size = 10
disk-type = virtio disk-type = virtio
cpu-count = 1 cpu-count = 1
# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus] cpu-max-count = 24
cpu-options =
# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
numa =
disk-cache = writeback disk-cache = writeback
disk-aio = native disk-aio = native
auto-ballooning = True auto-ballooning = True
......
...@@ -93,6 +93,7 @@ context = ...@@ -93,6 +93,7 @@ context =
raw netcat_bin ${netcat:location}/bin/netcat raw netcat_bin ${netcat:location}/bin/netcat
raw openssl_executable_location ${openssl:location}/bin/openssl raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_executable ${buildout:executable} raw python_executable ${buildout:executable}
raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64 raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
raw qemu_img_executable_location ${kvm:location}/bin/qemu-img raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename} raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}
......
...@@ -5,13 +5,13 @@ extends = common.cfg ...@@ -5,13 +5,13 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1 websockify = 0.5.1
slapos.toolbox = 0.73 slapos.toolbox = 0.74
erp5.util = 0.4.49 erp5.util = 0.4.49
apache-libcloud = 1.1.0 apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
gitdb = 0.6.4 gitdb = 0.6.4
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
smmap = 0.9.0 smmap = 0.9.0
# websockify = 0.8.0 # websockify = 0.8.0
......
...@@ -6,40 +6,58 @@ ...@@ -6,40 +6,58 @@
import socket import socket
import time import time
import os import os
from slapos.qemuqmpclient import QemuQMPWrapper, getInitialQemuResourceDict
# XXX: to be factored with slapos.toolbox qemu qmp wrapper. # XXX: to be factored with slapos.toolbox qemu qmp wrapper.
socket_path = '{{ parameter_dict.get("socket-path") }}' socket_path = '{{ parameter_dict.get("socket-path") }}'
pid_file = '{{ parameter_dict.get("pid-file") }}'
vnc_password = '{{ parameter_dict.get("vnc-passwd") }}' vnc_password = '{{ parameter_dict.get("vnc-passwd") }}'
status_path = '{{ parameter_dict.get("kvm-status-path") }}' status_path = '{{ parameter_dict.get("kvm-status-path") }}'
cpu_amount = {{ parameter_dict.get("cpu-count") }}
cpu_model = '{{ parameter_dict.get("cpu-model", "qemu64-x86_64-cpu") }}'
slot_hotplug_size = {{ parameter_dict.get("ram-hotplug-slot-size", 512) }}
ram_size = {{ parameter_dict.get("ram-size") }}
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
if os.path.exists(status_path): def write(message):
os.unlink(status_path) with open(status_path, 'w') as status_file:
status_file.write(message)
def update():
# Connect to KVM qmp socket
so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connected = False
while not connected:
try: try:
so.connect(socket_path) init_dict = getInitialQemuResourceDict(pid_file)
except socket.error: init_ram_size = int(init_dict['ram'].split('M')[0])
time.sleep(1) if cpu_amount < 1:
else: raise ValueError("CPU should be at least equal to 1.")
connected = True hotplug_ram = ram_size - init_ram_size
data = so.recv(1024) if hotplug_ram < 0:
raise ValueError("RAM size cannot be less than the initial value %s MB" % init_ram_size)
# Enable qmp if os.path.exists(status_path):
so.send('{ "execute": "qmp_capabilities" }') os.unlink(status_path)
data = so.recv(1024)
qemu_wrapper = QemuQMPWrapper(socket_path)
# Set VNC password qemu_wrapper.setVNCPassword(vnc_password)
so.send('{ "execute": "change", ' \ if enable_device_hotplug == 'true':
'"arguments": { "device": "vnc", "target": "password", ' \ write("Qemu Controller is still running...")
' "arg": "' + vnc_password + '" } }') qemu_wrapper.updateDevice({
data = so.recv(1024) 'device': 'cpu',
'amount': cpu_amount,
# Finish 'model': cpu_model
so.close() })
qemu_wrapper.updateDevice({
with open(status_path, 'w') as status_file: 'device': 'memory',
status_file.write("OK") 'mem': hotplug_ram,
'slot': slot_hotplug_size,
'nslot': 128,
'canreboot': 1
})
except Exception, e:
write(str(e))
raise
write("")
if __name__ == "__main__":
update()
#!{{ dash }} #!{{ dash }}
if [ -f "{{ qemu_ready_path }}" ]; then FILE="{{ qemu_ready_path }}"
# don't start checks too fast
sleep 2
if [ -f "$FILE" ]; then
if [ "$(cat $FILE)" = "" ]; then
echo "VM correctly started." echo "VM correctly started."
else
>&2 echo "Qemu Controller failed"
>&2 cat $FILE
exit 1
fi
else else
log_file="{{ qemu_service_log_file }}" log_file="{{ qemu_service_log_file }}"
>&2 echo "Qemu process is not correctly started." >&2 echo "Qemu process is not correctly started."
......
...@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}' ...@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
listen_ip = '{{ parameter_dict.get("ipv4") }}' listen_ip = '{{ parameter_dict.get("ipv4") }}'
mac_address = '{{ parameter_dict.get("mac-address") }}' mac_address = '{{ parameter_dict.get("mac-address") }}'
tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}' tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
smp_count = '{{ parameter_dict.get("smp-count") }}' numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
smp_options = '{{ parameter_dict.get("smp-options") }}'.strip() ram_size = {{ parameter_dict.get("ram-size") }}
numa_list = '{{ parameter_dict.get("numa") }}'.split() ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
ram_size = '{{ parameter_dict.get("ram-size") }}' init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}' pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }} external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = '{{ parameter_dict.get("external-disk-size") }}' external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format") }}' external_disk_format = '{{ parameter_dict.get("external-disk-format") }}'
disk_storage_dict = {} disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n') disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
...@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe", ...@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip() disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] and \ disk_aio = disk_aio if disk_aio in ["threads", "native"] and \
disk_cache == "directsync" else "threads" disk_cache == "directsync" else "threads"
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip() machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip() cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip()
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
logfile = '{{ parameter_dict.get("log-file") }}' logfile = '{{ parameter_dict.get("log-file") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
...@@ -255,15 +259,14 @@ if use_tap == 'true': ...@@ -255,15 +259,14 @@ if use_tap == 'true':
tap_interface, vhost), tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)] '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
smp = smp_count if enable_device_hotplug != 'true':
if smp_options: smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
for option in smp_options.split(','): ram = '%sM,slots=128,maxmem=%s' % (ram_size, ram_max_size)
key, val = option.split('=') else:
if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit(): smp = '1,maxcpus=%s' % smp_max_count
smp += ',%s=%s' % (key, val) ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path, kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-m', ram_size, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio), '-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
'-vnc', '%s:1,ipv4,password' % listen_ip, '-vnc', '%s:1,ipv4,password' % listen_ip,
'-boot', 'order=cd,menu=on', '-boot', 'order=cd,menu=on',
......
...@@ -17,7 +17,7 @@ parts += ...@@ -17,7 +17,7 @@ parts +=
versions = versions versions = versions
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
[template-instance] [template-instance]
......
...@@ -116,8 +116,8 @@ mysqlclient = 1.3.12 ...@@ -116,8 +116,8 @@ mysqlclient = 1.3.12
persistent = 4.2.3 persistent = 4.2.3
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap2 = 2.0.1 smmap2 = 2.0.1
transaction = 1.7.0 transaction = 1.7.0
zodbpickle = 0.6.0 zodbpickle = 0.6.0
......
# TODO instance which runs this test periodically automatically and ingests results to ERP5 # NEO test instance: run neotest under Nexedi testing infrastructure
[buildout]
parts = runTestSuite
# std stuff for slapos instance
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
# software release we instantiate was supplied here
[software]
dir = {{ buildout['directory'] }}
bin = {{ buildout['bin-directory'] }}
# instance directories
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
bin = ${:home}/bin
neotest = ${:home}/neotest
# script to run the testsuite from inside <instance>/neotest/
# located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
[runTestSuite]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
# vvv appends "$@" to argv ^^^ without shell-escaping
parameters-extra = true
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
. ${gowork:env.sh} . ${gowork:env.sh}
PATH="${coreutils:location}/bin:$PATH"
PATH="${util-linux:location}/bin:$PATH"
PATH="${ethtool:location}/sbin:$PATH" PATH="${ethtool:location}/sbin:$PATH"
PATH="${ioping:location}/bin:$PATH" PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH" PATH="${lmbench:location}/bin:$PATH"
......
#!/bin/bash -e
# neotest's runTestSuite wraper so it could be run without any environment preset
. ${buildout:directory}/neotest-env.sh
exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"
...@@ -12,6 +12,8 @@ extends = ...@@ -12,6 +12,8 @@ extends =
../../component/ethtool/buildout.cfg ../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg ../../component/ioping/buildout.cfg
../../component/lmbench/buildout.cfg ../../component/lmbench/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/util-linux/buildout.cfg
parts = parts =
gowork gowork
...@@ -19,6 +21,11 @@ parts = ...@@ -19,6 +21,11 @@ parts =
ioping ioping
ethtool ethtool
# neotest uses realpath & friends
coreutils
# ----//---- lsblk
util-linux
neoppod-develop neoppod-develop
neoppod neoppod
wendelin.core-dev wendelin.core-dev
...@@ -26,11 +33,12 @@ parts = ...@@ -26,11 +33,12 @@ parts =
neotest-env.sh neotest-env.sh
neotest neotest
neotest-runTestSuite
# for instance # for instance
slapos-deps-eggs slapos-deps-eggs
slapos-cookbook slapos-cookbook
# instance.cfg instance.cfg
# go packages to install (+ automatically their dependencies) # go packages to install (+ automatically their dependencies)
...@@ -53,7 +61,7 @@ output = ${buildout:directory}/${:_buildout_section_name_} ...@@ -53,7 +61,7 @@ output = ${buildout:directory}/${:_buildout_section_name_}
[neotest-env.sh] [neotest-env.sh]
<= buildout-template <= buildout-template
md5sum = 6e2203c7a86e8a3e8e56b8086115f5d4 md5sum = 595eebbbcb56b6a8464d48833cfae57b
[neotest] [neotest]
<= buildout-template <= buildout-template
...@@ -61,11 +69,25 @@ output = ${buildout:bin-directory}/${:_buildout_section_name_} ...@@ -61,11 +69,25 @@ output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755 mode = 0755
md5sum = fb3b4109128c1db1739ef5bb6abd1d94 md5sum = fb3b4109128c1db1739ef5bb6abd1d94
[neotest-runTestSuite]
<= buildout-template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = 6a4281730b68cdba5c873817a6754428
# instance
[jinja2-template]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
rendered= ${buildout:directory}/${:_buildout_section_name_}
mode = 0644
context =
section buildout buildout
# instance (TODO)
[instance.cfg] [instance.cfg]
<= buildout-template <= jinja2-template
md5sum = 410e1b2d72829824b28cc0299adb472e md5sum = 7bd68ed8842cd25301bf04bcdcef88f6
# eggs: # eggs:
...@@ -80,6 +102,8 @@ eggs = ...@@ -80,6 +102,8 @@ eggs =
wendelin.core wendelin.core
# for ZEO scripts (runzeo) # for ZEO scripts (runzeo)
ZEO ZEO
# for nxd/runTestSuite
erp5.util
# wendelin.core: latest not yet released # wendelin.core: latest not yet released
...@@ -92,6 +116,7 @@ pyasn1 = 0.3.7 ...@@ -92,6 +116,7 @@ pyasn1 = 0.3.7
ZODB3 = 3.11.0 ZODB3 = 3.11.0
numpy = 1.13.3 numpy = 1.13.3
zope.testing = 4.6.2 zope.testing = 4.6.2
erp5.util = 0.4.50
# Required by: # Required by:
# ZEO==4.3.1 # ZEO==4.3.1
...@@ -103,7 +128,3 @@ ZConfig = 3.2.0 ...@@ -103,7 +128,3 @@ ZConfig = 3.2.0
# ZEO==4.3.1 # ZEO==4.3.1
# ZODB==4.4.5 # ZODB==4.4.5
zc.lockfile = 1.2.1 zc.lockfile = 1.2.1
# Required by:
# slapos.toolbox==0.73
erp5.util = 0.4.49
...@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in ...@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
dnspython = 1.15.0
PyRSS2Gen = 1.1
erp5.util = 0.4.50
passlib = 1.7.1
GitPython = 2.1.8
lockfile = 0.12.2
apache-libcloud = 2.2.1
feedparser = 5.2.1
atomize = 0.2.0
inotifyx = 0.2.2
gitdb2 = 2.0.3
pyasn1 = 0.4.2
smmap2 = 2.0.3
...@@ -65,4 +65,4 @@ mode = 0644 ...@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -110,8 +110,8 @@ gitdb = 0.6.4 ...@@ -110,8 +110,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -47,4 +47,4 @@ mode = 0644 ...@@ -47,4 +47,4 @@ mode = 0644
[versions] [versions]
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -11,4 +11,4 @@ extends = common.cfg ...@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6 Pygments = 1.6
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10 collective.recipe.template = 1.10
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -97,4 +97,4 @@ mode = 640 ...@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3 Pygments = 2.1.3
collective.recipe.template = 1.10 collective.recipe.template = 1.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab ...@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
[template-runner] [template-runner]
filename = instance-runner.cfg filename = instance-runner.cfg
md5sum = 7df86928aeef0fbada832b0283b9a454 md5sum = 8b1caca52ab1307343ebada59f5a6c66
[template-runner-import-script] [template-runner-import-script]
filename = template/runner-import.sh.jinja2 filename = template/runner-import.sh.jinja2
...@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb ...@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
[instance-runner-export] [instance-runner-export]
filename = instance-runner-export.cfg.in filename = instance-runner-export.cfg.in
md5sum = 00ddcaa6e571505d1930612d12de8fde md5sum = d1122988652127af5e252f1503eeacab
[template-resilient] [template-resilient]
filename = instance-resilient.cfg.jinja2 filename = instance-resilient.cfg.jinja2
......
...@@ -7,11 +7,13 @@ parts += ...@@ -7,11 +7,13 @@ parts +=
nginx-launcher nginx-launcher
certificate-authority certificate-authority
ca-nginx ca-nginx
logrotate-entry-nginx
gunicorn-launcher gunicorn-launcher
gunicorn-graceful gunicorn-graceful
publish-connection-information publish-connection-information
slaprunner-promise slaprunner-promise
apache-httpd-promise apache-httpd-promise
logrotate-entry-apache-httpd
slaprunner-supervisord-wrapper slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key runner-sshd-add-authorized-key
runner-sshd-graceful runner-sshd-graceful
...@@ -30,6 +32,7 @@ parts += ...@@ -30,6 +32,7 @@ parts +=
bash-profile bash-profile
supervisord-wrapper supervisord-wrapper
supervisord-promise supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper httpd-graceful-wrapper
## Monitoring part ## Monitoring part
## Monitor for runner ## Monitor for runner
......
...@@ -4,10 +4,12 @@ parts = ...@@ -4,10 +4,12 @@ parts =
nginx-launcher nginx-launcher
certificate-authority certificate-authority
ca-nginx ca-nginx
logrotate-entry-nginx
gunicorn-launcher gunicorn-launcher
gunicorn-graceful gunicorn-graceful
publish-connection-information publish-connection-information
slaprunner-promise slaprunner-promise
logrotate-entry-apache-httpd
apache-httpd-promise apache-httpd-promise
slaprunner-supervisord-wrapper slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key runner-sshd-add-authorized-key
...@@ -27,6 +29,7 @@ parts = ...@@ -27,6 +29,7 @@ parts =
bash-profile bash-profile
supervisord-wrapper supervisord-wrapper
supervisord-promise supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper httpd-graceful-wrapper
{% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %} {% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %}
slaprunner-frontend-promise slaprunner-frontend-promise
...@@ -347,6 +350,12 @@ mode = 700 ...@@ -347,6 +350,12 @@ mode = 700
context = context =
section param_nginx_frontend nginx-frontend section param_nginx_frontend nginx-frontend
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = $${directory:log}/nginx.access.log $${directory:log}/nginx.error.log
post = kill -SIGUSR1 $(cat $${buildout:directory}/var/run/nginx.pid)
[httpd-parameters] [httpd-parameters]
path_pid = $${directory:run}/httpd.pid path_pid = $${directory:run}/httpd.pid
path_error_log = $${directory:log}/httpd-error.log path_error_log = $${directory:log}/httpd-error.log
...@@ -387,6 +396,12 @@ wait-for-files = ...@@ -387,6 +396,12 @@ wait-for-files =
$${ca-nginx:cert-file} $${ca-nginx:cert-file}
$${ca-nginx:key-file} $${ca-nginx:key-file}
[logrotate-entry-apache-httpd]
<= logrotate-entry-base
name = apache
log = $${directory:log}/httpd-access.log $${directory:log}/httpd-error.log
post = test ! -s $${buildout:directory}/var/run/httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
[httpd-graceful-wrapper] [httpd-graceful-wrapper]
recipe = collective.recipe.template recipe = collective.recipe.template
input = inline: input = inline:
...@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper ...@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper
command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
wrapper-path = $${directory:services}/supervisord wrapper-path = $${directory:services}/supervisord
[logrotate-entry-supervisord]
<= logrotate-entry-base
name = supervisord
log = $${directory:log}/slapproxy.log $${directory:log}/supervisord.log $${directory:log}/supervisord-errors.log
post = kill -SIGUSR2 $(cat $${buildout:directory}/var/run/supervisord.pid)
[supervisord-promise] [supervisord-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/supervisord path = $${directory:promises}/supervisord
......
...@@ -15,8 +15,8 @@ gitdb = 0.6.4 ...@@ -15,8 +15,8 @@ gitdb = 0.6.4
gunicorn = 19.7.1 gunicorn = 19.7.1
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -59,7 +59,7 @@ eggs = collective.recipe.template ...@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11 collective.recipe.template = 1.11
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.28 slapos.recipe.build = 0.28
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test. # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout] [buildout]
......
...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb ...@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640 mode = 640
[versions] [versions]
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
...@@ -94,8 +94,8 @@ caucase = 0.1.4 ...@@ -94,8 +94,8 @@ caucase = 0.1.4
futures = 3.1.1 futures = 3.1.1
gitdb2 = 2.0.2 gitdb2 = 2.0.2
gunicorn = 19.7.1 gunicorn = 19.7.1
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
slapos.toolbox = 0.73 slapos.toolbox = 0.74
smmap2 = 2.0.3 smmap2 = 2.0.3
# Required by: # Required by:
......
...@@ -89,7 +89,7 @@ PasteScript = 2.0.2 ...@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7 WSGIUtils = 0.7
python-magic = 0.4.6 python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2 slapos.recipe.template = 4.3
# Required by: # Required by:
# PasteScript==2.0 # PasteScript==2.0
......
...@@ -63,6 +63,7 @@ extends = ...@@ -63,6 +63,7 @@ extends =
../../component/findutils/buildout.cfg ../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg ../../component/userhosts/buildout.cfg
../../component/postfix/buildout.cfg ../../component/postfix/buildout.cfg
../../component/zbarlight/buildout.cfg
../monitor/buildout.cfg ../monitor/buildout.cfg
../../software/caucase/software.cfg ../../software/caucase/software.cfg
../../software/jupyter/software.cfg ../../software/jupyter/software.cfg
...@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs} ...@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs}
${h5py:egg} ${h5py:egg}
openpyxl openpyxl
${statsmodels:egg} ${statsmodels:egg}
${zbarlight:egg}
lock_file lock_file
astor astor
APacheDEX APacheDEX
...@@ -730,7 +732,7 @@ spyne = 2.12.14 ...@@ -730,7 +732,7 @@ spyne = 2.12.14
suds = 0.4 suds = 0.4
facebook-sdk = 2.0.0 facebook-sdk = 2.0.0
threadframe = 0.2 threadframe = 0.2
timerserver = 2.0.2 timerserver = 2.0.4
urlnorm = 1.1.4 urlnorm = 1.1.4
uuid = 1.30 uuid = 1.30
validictory = 1.1.0 validictory = 1.1.0
......
...@@ -61,7 +61,7 @@ eggs = ...@@ -61,7 +61,7 @@ eggs =
# Monitor templates files # Monitor templates files
[monitor-httpd-conf] [monitor-httpd-conf]
<= monitor-template-base <= monitor-template-base
md5sum = f2d6951670733de3b37c0ebe9eee343b md5sum = b5f42503799e7e770afce4097d3b75ae
filename = monitor-httpd.conf.in filename = monitor-httpd.conf.in
[monitor-template-wrapper] [monitor-template-wrapper]
...@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d ...@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
[monitor-conf] [monitor-conf]
<= monitor-template-base <= monitor-template-base
filename = monitor.conf.in filename = monitor.conf.in
md5sum = fc65084dd1c2fe2487b58a003b576f61 md5sum = 888e2845d09bfaa59c25f56f5bcf76b1
[monitor-instance-info] [monitor-instance-info]
<= monitor-template-base <= monitor-template-base
...@@ -131,6 +131,6 @@ depends = ...@@ -131,6 +131,6 @@ depends =
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
pycurl = 7.43.0 pycurl = 7.43.0
slapos.toolbox = 0.73 slapos.toolbox = 0.74
pyasn1 = 0.3.7 pyasn1 = 0.3.7
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
# not need these here). # not need these here).
[monitor2-template] [monitor2-template]
filename = instance-monitor.cfg.jinja2.in filename = instance-monitor.cfg.jinja2.in
md5sum = 5027f0b1abcc28ce3817cd70fb667a3b md5sum = 03254b14a2ff242f7588a307d8c27f23
...@@ -42,6 +42,7 @@ services = ${directory:services} ...@@ -42,6 +42,7 @@ services = ${directory:services}
services-conf = ${directory:etc}/monitor.conf.d services-conf = ${directory:etc}/monitor.conf.d
log = ${directory:log}/monitor log = ${directory:log}/monitor
monitor-var = ${directory:var}/monitor monitor-var = ${directory:var}/monitor
monitor-log = ${directory:monitor}/private/monitor-log
[ca-directory] [ca-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -88,14 +89,11 @@ promise-folder = ${directory:promises} ...@@ -88,14 +89,11 @@ promise-folder = ${directory:promises}
monitor-promise-folder = ${monitor-directory:promises} monitor-promise-folder = ${monitor-directory:promises}
promises-timeout-file = ${monitor-promise-timeout-file:file} promises-timeout-file = ${monitor-promise-timeout-file:file}
pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
randomsleep = {{ bin_directory }}/randomsleep
public-path-list = public-path-list =
private-path-list = private-path-list = ${directory:log}
${directory:log} monitor-url-list = ${monitor-instance-parameter:monitor-url-list}
#
monitor-url-list =
${monitor-instance-parameter:monitor-url-list}
parameter-file-path = ${monitor-instance-parameter:configuration-file-path} parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
parameter-list = parameter-list =
...@@ -234,6 +232,12 @@ context = ...@@ -234,6 +232,12 @@ context =
raw dash_binary {{ dash_executable_location }} raw dash_binary {{ dash_executable_location }}
command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file}) command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[logrotate-entry-monitor-httpd]
<= logrotate-entry-base
name = monitor-apache
log = $${basedirectory:log}/monitor/monitor-httpd-error.log
post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1
[xnice-bin] [xnice-bin]
recipe = collective.recipe.template recipe = collective.recipe.template
input = inline:#!/bin/sh input = inline:#!/bin/sh
...@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d ...@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries} cron-entries = ${cron:cron-entries}
name = monitor-globalstate name = monitor-globalstate
frequency = * * * * * frequency = * * * * *
command = ${monitor-globalstate-wrapper:wrapper-path} command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path}
[monitor-configurator-cron-entry] [monitor-configurator-cron-entry]
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries} cron-entries = ${cron:cron-entries}
name = monitor-configurator name = monitor-configurator
frequency = * * * * * frequency = * * * * *
command = ${monitor-configurator-wrapper:wrapper-path} command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path}
[monitor-httpd-promise] [monitor-httpd-promise]
recipe = slapos.cookbook:check_url_available recipe = slapos.cookbook:check_url_available
...@@ -407,6 +411,7 @@ depends = ...@@ -407,6 +411,7 @@ depends =
${monitor-bootstrap-promise:file} ${monitor-bootstrap-promise:file}
${promise-check-slapgrid:output} ${promise-check-slapgrid:output}
${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path} ${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
${logrotate-entry-monitor-httpd:name}
[monitor-publish] [monitor-publish]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
......
...@@ -2,6 +2,7 @@ PidFile "{{ parameter_dict.get('pid-file') }}" ...@@ -2,6 +2,7 @@ PidFile "{{ parameter_dict.get('pid-file') }}"
StartServers 1 StartServers 1
ServerLimit 1 ServerLimit 1
MaxRequestWorkers 4
ThreadLimit 4 ThreadLimit 4
ThreadsPerChild 4 ThreadsPerChild 4
......
[monitor] [monitor]
{% set monitor_url_list = parameter_dict.pop("monitor-url-list", "").strip() -%}
{% for key, value in parameter_dict.items() -%} {% for key, value in parameter_dict.items() -%}
{% if key == "monitor-url-list" and monitor_base_urls is defined -%} {{ key }} = {{ value.strip().replace("\n", "\n ") }}
{{ key }} = {% endfor -%}
monitor-url-list =
{% if monitor_url_list -%}
{{ ' ' ~ monitor_url_list.replace("\n", "\n ") }}
{% else -%}
{% for key, value in monitor_base_urls.items() -%} {% for key, value in monitor_base_urls.items() -%}
{{ ' ' ~ value }} {{ ' ' ~ value }}
{% endfor -%} {% endfor -%}
{% else -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endif -%} {% endif -%}
{% endfor -%}
...@@ -93,7 +93,7 @@ eggs = ...@@ -93,7 +93,7 @@ eggs =
[versions] [versions]
setuptools = 33.1.1 setuptools = 33.1.1
# Use SlapOS patched zc.buildout # Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos011 zc.buildout = 2.5.2+slapos012
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2) # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003 zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives # Use own version of h.r.download to be able to open .xz and .lz archives
...@@ -121,14 +121,14 @@ netaddr = 0.7.19 ...@@ -121,14 +121,14 @@ netaddr = 0.7.19
pbr = 2.0.0 pbr = 2.0.0
plone.recipe.command = 1.1 plone.recipe.command = 1.1
prettytable = 0.7.2 prettytable = 0.7.2
psutil = 5.2.0 psutil = 5.4.3
pyOpenSSL = 17.2.0 pyOpenSSL = 17.2.0
pyparsing = 2.2.0 pyparsing = 2.2.0
pytz = 2016.10 pytz = 2016.10
requests = 2.13.0 requests = 2.13.0
six = 1.10.0 six = 1.10.0
slapos.cookbook = 1.0.53 slapos.cookbook = 1.0.53
slapos.core = 1.4.3 slapos.core = 1.4.4
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15 slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1 slapos.rebootstrap = 4.1
...@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7 ...@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7
paramiko = 2.1.3 paramiko = 2.1.3
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
Flask = 0.12 Flask = 0.12
# Required by: # Required by:
...@@ -160,7 +160,7 @@ ipaddress = 1.0.18 ...@@ -160,7 +160,7 @@ ipaddress = 1.0.18
jsonschema = 2.6.0 jsonschema = 2.6.0
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
# XXX 'slapos node format' raises an exception with netifaces 0.10.5. # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4 netifaces = 0.10.4
...@@ -173,15 +173,15 @@ packaging = 16.8 ...@@ -173,15 +173,15 @@ packaging = 16.8
pycparser = 2.17 pycparser = 2.17
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
supervisor = 3.3.3 supervisor = 3.3.3
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
uritemplate = 3.0.0 uritemplate = 3.0.0
# Required by: # Required by:
# slapos.core==1.4.3 # slapos.core==1.4.4
zope.interface = 4.3.3 zope.interface = 4.3.3
[networkcache] [networkcache]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment