Commit 58f2c3a6 authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 009513b1 9437ec3a
......@@ -2,6 +2,7 @@
extends =
../coreutils/buildout.cfg
../patch/buildout.cfg
../randomsleep/buildout.cfg
parts = dcron-output
......@@ -18,6 +19,7 @@ make-options =
post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook
environment =
PATH=${patch:location}/bin:%(PATH)s
randomsleep_install = ${randomsleep:location}
[dcron-output]
# Shared binary location to ease migration
......
......@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location}
[golang18]
<= golang-common
url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz
md5sum = 64e9380e07bba907e26a00cf5fcbe77e
url = https://dl.google.com/go/go1.8.7.src.tar.gz
md5sum = c61cfe9c85e7d42f903d3fe146d7cde6
# go1.8 needs go1.4 to bootstrap
environment-extra =
......@@ -43,8 +43,8 @@ environment-extra =
[golang19]
<= golang-common
url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz
md5sum = 44105c865a1a810464df79233a05a568
url = https://dl.google.com/go/go1.9.4.src.tar.gz
md5sum = 6816441fd6680c63865cdd5cb8bc1960
# go1.9 needs go1.4 to bootstrap
environment-extra =
......@@ -63,6 +63,11 @@ environment-extra =
# lab.nexedi.com/kirr/neo/go/... \
# github.com/pkg/profile \
# golang.org/x/perf/cmd/benchstat
#
# it is possible to specify Go build flags used for compilation e.g. this way:
#
# [gowork]
# buildflags = -race
[gowork]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
......@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications)
golang = ${golang19:location}
# no special build flags by default
buildflags =
# everything is done by dependent parts
recipe = plone.recipe.command
command = :
......@@ -100,7 +108,7 @@ stop-on-error = true
# clients should put package list to install to gowork:install ("..." requests installing everything)
[gowork.goinstall]
recipe = plone.recipe.command
command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}"
command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v ${gowork:install}"
update-command = ${:command}
stop-on-error = true
......@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git
# 3) provide repository (which is not the same as importpath in general case)
#
# the list of go packages for a go workspace state can be automatically
# generated with the help of go-pkg-snapshot tool.
# generated with the help of gowork-snapshot tool.
[go-git-package]
<= git-repository
location = ${gowork:src}/${:go.importpath}
......@@ -27,8 +27,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.1.30
md5sum = c424fd12bdff388e3da1bdecf42626c9
version = 10.1.31
md5sum = 14ab0398c019eb531bc29f2c437ccb51
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......
From efa86aa5ad5b89ec329b3cde288cb119efb21c89 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Thu, 06 Jul 2017 21:15:31 +0200
Subject: [PATCH] asyncore: switch default to use poll instead of select
Recently we started to get the following errors on a Wendelin production
instance:
File ".../bin/runzope", line 211, in <module>
sys.exit(Zope2.Startup.run.run())
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/run.py", line 26, in run
starter.run()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/__init__.py", line 111, in run
Lifetime.loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 43, in loop
lifetime_loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 53, in lifetime_loop
asyncore.poll(timeout, map)
File ".../parts/python2.7/lib/python2.7/asyncore.py", line 145, in poll
r, w, e = select.select(r, w, e, timeout)
ValueError: filedescriptor out of range in select()
Initially we thought the reason here is that number of file descriptors this
process uses goes beyond allowed limit (65K open files on that instance)
because wendelin.core uses 1 fd per an array view (opening
/dev/shm/ramh.XXXXX) but that turned out to be not strictly that case:
The reason here is that select() has limit for how many #fd it can
monitor at all. The limit is system-specific but on Linux it is usually
1024 - http://man7.org/linux/man-pages/man2/select.2.html#BUGS :
$ cat s.c
#include <sys/select.h>
#include <stdio.h>
int main() {
printf("%d\n", FD_SETSIZE);
return 0;
}
$ tcc -run s.c
1024
Also select() can monitor only file descriptors which are by itself
"< FD_SETSIZE", e.g. it cannot monitor fd=1025 even if there are only 10
opened file descriptors because 1025 is not < 1024.
As was said above in wendelin.core every array view uses 1 file
descriptor, so if we are using not so small amount of arrays, even
though #fd does not go beyond process-specific ulimit, because of
select() usage the total number of allowed opened file descriptors
becomes essentially 1024.
So let's switch from select() to poll() which does not have this 1024
#fd limit. Asyncore already support using poll() out of the box -
either via passing use_pull=True to asyncore.loop()
https://docs.python.org/2/library/asyncore.html#asyncore.loop
or by using asyncore.poll2() instead of asyncore.poll()
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L170
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L125
--------
One option would be to switch asyncore.poll() -> asyncore.poll2() for only 1
place in Zope - inside lifetime_loop(). There are however many such similar
places in Zope (e.g. in medusa) and in other software.
For this reason, for me, what makes sense to do is not to patch all such
places, but instead change via runtime-patching asyncore.poll to be
asyncore.poll2 - this way all software will automatically benefit from
poll() usage instead of select.
P.S.
What might also make sense to do in the future is to actually let
asyncore.poll to use epoll(), as both select() and poll() are doing all
fd registration on _every_ call, so when #fd grows this preparation
time grows too. For epoll() file descriptors are registered only once.
For this to work asyncore.socket_map has to be patched also, since there
are places in code which modify this fd registry directly (e.g. remove
fd from there etc)
Original patch & discussion: https://lab.nexedi.com/kirr/Zope/commit/c6addb05
---
Lib/asyncore.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 105982f..a3025a4 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -661,3 +661,12 @@ if os.name == 'posix':
self.socket = file_wrapper(fd)
self._fileno = self.socket.fileno()
self.add_channel()
+
+# monkeypatch to switch everyone's default to use poll instead of select
+# (select has 1024 fd limit)
+# TODO it is better to use epoll
+_poll_select = poll
+_poll_poll = poll2
+poll = _poll_poll
+
+#print >>sys.stderr, 'I(nxd): asyncore: using poll instead of select'
--
2.16.1.101.gde0f0111ea
......@@ -43,6 +43,7 @@ patches =
${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a
${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
url =
http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
configure-options =
......
......@@ -19,12 +19,11 @@ extends =
[kvm]
recipe = slapos.recipe.cmmi
# qemu-kvm and qemu are now the same since 1.3.
url = http://wiki.qemu-project.org/download/qemu-2.8.0.tar.bz2
md5sum = 17940dce063b6ce450a12e719a6c9c43
url = http://wiki.qemu-project.org/download/qemu-2.11.0.tar.bz2
md5sum = 335994a755bc655e88a87aeb36bfc0b9
configure-options =
--target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
--enable-system
--with-system-pixman
--disable-sdl
--disable-xen
--disable-vnc-sasl
......@@ -58,22 +57,18 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
<= debian-netinst-base
arch = amd64
[debian-amd64-squeeze-netinst.iso]
<= debian-amd64-netinst-base
version = 6.0.10
md5sum = 7f82d341561035f65933da43f94d5b52
[debian-amd64-wheezy-netinst.iso]
[debian-amd64-jessie-netinst.iso]
# Download the installer of Debian 8 (Jessie)
<= debian-amd64-netinst-base
version = 7.11.0
md5sum = 096c1c18b44c269808bd815d58c53c8f
release = archive
version = 8.10.0
md5sum = 19dcfc381bd3e609c6056216d203f5bc
[debian-amd64-netinst.iso]
# Download the installer of Debian 8 (Jessie)
<= debian-amd64-netinst-base
release = release/current
version = 8.6.0
md5sum = e9f61bf327db6d8f7cee05a99f2353cc
version = 9.3.0
md5sum = db8ab7871bc2b7d456c4746e706fb5d3
[debian-amd64-testing-netinst.iso]
# Download the installer of Debian Stretch
......
[buildout]
extends =
../coreutils/buildout.cfg
../bash/buildout.cfg
[randomsleep]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin_dir = ${buildout:bin-directory}
bash_script_code =
if [ "$#" -ne 1 ]; then
echo "usage: randomsleep maxseconds"
exit
fi
${coreutils:location}/bin/sleep $((RANDOM % $1)).$((RANDOM % 100))
wrapper_script_code =
#!${bash:location}/bin/bash
${bash:location}/bin/bash ${:location}/randomsleep.bash "$@"
script =
os.makedirs(self.options['location'])
bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
with open(bash_script_path, 'w') as f:
f.write(self.options['bash_script_code'])
wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
with open(wrapper_script_path, 'w') as f:
f.write(self.options['wrapper_script_code'])
os.chmod(wrapper_script_path, 0750)
diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
index b386d59..f03b056 100755
index 88fb5a2..782dbc5 100755
--- a/tensorboard/pip_package/build_pip_package.sh
+++ b/tensorboard/pip_package/build_pip_package.sh
@@ -26,6 +26,7 @@ function main() {
......@@ -10,14 +10,14 @@ index b386d59..f03b056 100755
echo $(date) : "=== Using tmpdir: ${TMPDIR}"
@@ -45,8 +46,8 @@ function main() {
@@ -44,9 +45,7 @@ function main() {
pushd ${TMPDIR} >/dev/null
rm -f MANIFEST
echo $(date) : "=== Building wheel"
echo $(pwd)
- python setup.py bdist_wheel >/dev/null
- python3 setup.py bdist_wheel >/dev/null
echo $(date) : "=== Building python2 wheel in $PWD"
- python setup.py bdist_wheel --python-tag py2 >/dev/null
- echo $(date) : "=== Building python3 wheel in $PWD"
- python setup.py bdist_wheel --python-tag py3 >/dev/null
+ PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
+ #python3 setup.py bdist_wheel >/dev/null
mkdir -p ${DEST}
cp dist/* ${DEST}
popd
popd >/dev/null
......@@ -11,11 +11,11 @@ parts =
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/tensorflow/tensorboard
tag = 0.4
tag = 0.4.0
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1)
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
[tensorboard-build]
recipe = slapos.recipe.build
......@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script =
os.makedirs(location)
workdir = self.options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg'
egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'],
......
......@@ -5,8 +5,8 @@ extends =
[util-linux]
recipe = slapos.recipe.cmmi
url = http://www.kernel.org/pub/linux/utils/util-linux/v2.29/util-linux-2.29.tar.xz
md5sum = 07b6845f48a421ad5844aa9d58edb837
url = https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.1.tar.xz
md5sum = 7733b583dcb51518944d42aa62ef19ea
configure-options =
--disable-static
--enable-libuuid
......@@ -29,7 +29,7 @@ configure-options =
--disable-rename
--disable-schedutils
--disable-setterm
--disable-libsmartcols
--enable-libsmartcols
--disable-switch_root
--disable-tinfo
--disable-tls
......
[buildout]
parts=
zbar
[zbar]
recipe = slapos.recipe.cmmi
url = https://jaist.dl.sourceforge.net/project/zbar/zbar/0.10/zbar-0.10.tar.bz2
md5sum = 0fd61eb590ac1bab62a77913c8b086a5
configure-options =
--disable-video
--without-imagemagick
--without-gtk
--without-xv
--without-qt
--without-python
--without-x
--without-jpg
environment =
CFLAGS=
[buildout]
extends =
../pillow/buildout.cfg
../zbar/buildout.cfg
parts =
zbarlight
[zbarlight]
recipe = zc.recipe.egg:custom
egg = zbarlight
include-dirs =
${zbar:location}/include
library-dirs =
${zbar:location}/lib
rpath =
${zbar:location}/lib
depend-eggs = ${pillow-python:egg}
......@@ -50,8 +50,8 @@ gitdb = 0.6.4
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
# Required by:
......
......@@ -10,8 +10,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......
......@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
erp5.util = 0.4.49
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies
cp.recipe.cmd = 0.5
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
zope.exceptions = 4.0.7
zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0
......
......@@ -117,7 +117,7 @@
"timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"default": 5,
"type": "integer"
"type": "number"
},
"ssl-authentication": {
"title": "Enable SSL Client authentication on this zope instance.",
......
......@@ -59,7 +59,7 @@ mode = 0644
[versions]
PyXML = 0.8.5
erp5.util = 0.4.50
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
ipython = 5.3.0
apache-libcloud = 2.1.0
gitdb2 = 2.0.2
......
......@@ -106,6 +106,7 @@ def waitForSite(partition_path):
date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration=end - start,
)
print(try_info + 'status_dict %r' % (status_dict,))
return status_dict
......
......@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644
[versions]
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
z3c.recipe.scripts = 1.0.1
......@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4
pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
slapos.toolbox = 0.40.4
smmap = 0.8.2
plone.recipe.command = 1.1
......
......@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -113,5 +113,5 @@ mode = 0644
[versions]
erp5.util = 0.4.50
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
selenium = 3.8.0
......@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1
seaborn = 0.7.1
simplegeneric = 0.8.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
statsmodels = 0.8.0
terminado = 0.6
tornado = 4.4.2
......
......@@ -41,6 +41,7 @@ parts = ${:common-parts}
[eggs]
recipe = zc.recipe.egg
interpreter = python.eggs
eggs =
${python-cffi:egg}
${python-cryptography:egg}
......@@ -89,7 +90,7 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = bf5ef731c0d8da0267a4939882b4eeee
md5sum = 5a17fc127190bbc19361c5ffb10711b3
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644
md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d
md5sum = e3cc9ffe857da1078e321cab65173fb1
download-only = true
on-update = true
......@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644
md5sum = 05b6004e8c7a94de14f247affcef4971
md5sum = 26d931a0279d49eadb3881f440b623bc
download-only = true
on-update = true
......@@ -185,7 +186,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/template-kvm-run.in
mode = 644
filename = template-kvm-run.in
md5sum = 178a24cdad77cb6c2e519ac629dd0e74
md5sum = 122bf5e8c7a12bceef1bdd6d6b54f4d7
download-only = true
on-update = true
......@@ -195,7 +196,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/kvm-controller-run.in
mode = 644
filename = kvm-controller-run.in
md5sum = 71afd2d13f6e56993ae413a168e012d7
md5sum = 7e6c79232cc88c15ed21c112ff801b76
download-only = true
on-update = true
......@@ -225,7 +226,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/qemu-is-ready.in
mode = 644
filename = qemu-is-ready.in
md5sum = 0066fa0f5f3dd47bded5e5924df2550d
md5sum = b304eec8e2cb71f10ea83cac22f6db12
download-only = true
on-update = true
......
......@@ -189,13 +189,35 @@
"default": "started",
"enum": ["started", "stopped"]
},
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": {
"title": "RAM size",
"description": "RAM size, in MB.",
"type": "integer",
"default": 1024,
"minimum": 128,
"multipleOf": 128
"minimum": 1024,
"multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
},
"auto-ballooning": {
"title": "Enable qemu auto ballooning.",
......@@ -245,10 +267,13 @@
"type": "integer",
"minimum": 1
},
"cpu-options": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
"type": "string"
"cpu-max-count": {
"title": "Maximum CPU amount",
"description": "Specifies the maximum number of CPUs.",
"type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
},
"numa": {
"title": "Simulate a multi node NUMA system.",
......
......@@ -8,7 +8,7 @@
{% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
{% set kvm_instance_dict = {} -%}
{% set kvm_hostname_list = [] -%}
{% set monitor_url_list = [] -%}
{% set monitor_base_url_dict = {} -%}
{% macro setconfig(name, value) -%}
{# will set a config-name = value if value is not empty -#}
......@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', '50G')) }}
config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', 24)) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
{{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
{{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}
......@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
{{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
{{ setconfig('nat-rules', ' '.join(nat_rules_list)) }}
{{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
config-publish-nat-url = True
config-use-nat = {{ use_nat }}
config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
......@@ -124,10 +127,10 @@ return =
{% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%}
{{ ' ' }}tap-ipv4
{% do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% endif -%}
{% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
{% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}
......@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of
username = admin
password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters]
monitor-url-list +=
{% for url in monitor_url_list -%}
{{ ' ' ~ url }}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[monitor-conf-parameters]
private-path-list +=
${directory:webroot}/
......
......@@ -4,13 +4,35 @@
"title": "Input Parameters",
"properties": {
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": {
"title": "RAM size",
"description": "RAM size, in MB.",
"type": "integer",
"default": 1024,
"minimum": 128,
"multipleOf": 128
"minimum": 1024,
"multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
},
"auto-ballooning": {
"title": "Enable qemu auto ballooning.",
......@@ -54,10 +76,25 @@
"type": "integer",
"minimum": 1
},
"cpu-options": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
"type": "string"
"cpu-max-count": {
"title": "Maximum CPU amount",
"description": "Specifies the maximum number of CPUs.",
"type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
},
"max-cpu-hotplug-count": {
"title": "Maximum hotpluggable CPU amount",
"description": "Specifies the maximum number of hotpluggable CPUs.",
"type": "integer",
"default": 1
},
"cpu-hotplug-amount": {
"title": "CPU hotplug amount",
"description": "Specifies the number of CPUs to hotplug",
"type": "integer",
"default": 0
},
"numa": {
"title": "Simulate a multi node NUMA system.",
......
......@@ -5,6 +5,7 @@
{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
{% set name = slapparameter_dict.get('name', 'localhost') -%}
{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set frontend_software_type = 'default' -%}
......@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd
bytes = 8
[kvm-controller-parameter-dict]
python-path = {{ python_executable }}
python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd}
socket-path = ${directory:var}/qmp_socket
pid-file = ${directory:run}/pid_file
kvm-status-path = ${directory:var}/qemu-vm-is-ready
cpu-count = ${slap-parameter:cpu-count}
ram-hotplug-slot-size = ${slap-parameter:ram-hotplug-slot-size}
ram-size = ${slap-parameter:ram-size}
enable-device-hotplug = ${slap-parameter:enable-device-hotplug}
[kvm-parameter-dict]
python-path = {{ python_executable }}
......@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size}
disk-type = ${slap-parameter:disk-type}
pid-file-path = ${directory:run}/pid_file
pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path}
smp-count = ${slap-parameter:cpu-count}
smp-options = ${slap-parameter:cpu-options}
ram-size = ${slap-parameter:ram-size}
numa = ${slap-parameter:numa}
enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-count = ${kvm-controller-parameter-dict:cpu-count}
smp-max-count = ${slap-parameter:cpu-max-count}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = ${slap-parameter:ram-max-size}
init-ram-size = 1024
mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address}
......@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache}
disk-aio = ${slap-parameter:disk-aio}
auto-ballooning = ${slap-parameter:auto-ballooning}
machine-options = ${slap-parameter:machine-options}
cpu-options = ${slap-parameter:cpu-model}
cpu-hotplug-slot-size = ${slap-parameter:cpu-model}
log-file = ${directory:log}/qemu.log
......@@ -550,15 +559,15 @@ nbd-host =
nbd2-port = 1024
nbd2-host =
enable-device-hotplug = False
ram-size = 1024
ram-max-size = 50G
ram-hotplug-slot-size = 512
disk-size = 10
disk-type = virtio
cpu-count = 1
# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
cpu-options =
# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
numa =
cpu-max-count = 24
disk-cache = writeback
disk-aio = native
auto-ballooning = True
......
......@@ -93,6 +93,7 @@ context =
raw netcat_bin ${netcat:location}/bin/netcat
raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_executable ${buildout:executable}
raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}
......
......@@ -5,13 +5,13 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
slapos.toolbox = 0.73
slapos.toolbox = 0.74
erp5.util = 0.4.49
apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
smmap = 0.9.0
# websockify = 0.8.0
......
......@@ -6,40 +6,58 @@
import socket
import time
import os
from slapos.qemuqmpclient import QemuQMPWrapper, getInitialQemuResourceDict
# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
socket_path = '{{ parameter_dict.get("socket-path") }}'
pid_file = '{{ parameter_dict.get("pid-file") }}'
vnc_password = '{{ parameter_dict.get("vnc-passwd") }}'
status_path = '{{ parameter_dict.get("kvm-status-path") }}'
cpu_amount = {{ parameter_dict.get("cpu-count") }}
cpu_model = '{{ parameter_dict.get("cpu-model", "qemu64-x86_64-cpu") }}'
slot_hotplug_size = {{ parameter_dict.get("ram-hotplug-slot-size", 512) }}
ram_size = {{ parameter_dict.get("ram-size") }}
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
if os.path.exists(status_path):
os.unlink(status_path)
def write(message):
with open(status_path, 'w') as status_file:
status_file.write(message)
def update():
# Connect to KVM qmp socket
so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connected = False
while not connected:
try:
so.connect(socket_path)
except socket.error:
time.sleep(1)
else:
connected = True
data = so.recv(1024)
# Enable qmp
so.send('{ "execute": "qmp_capabilities" }')
data = so.recv(1024)
# Set VNC password
so.send('{ "execute": "change", ' \
'"arguments": { "device": "vnc", "target": "password", ' \
' "arg": "' + vnc_password + '" } }')
data = so.recv(1024)
# Finish
so.close()
with open(status_path, 'w') as status_file:
status_file.write("OK")
init_dict = getInitialQemuResourceDict(pid_file)
init_ram_size = int(init_dict['ram'].split('M')[0])
if cpu_amount < 1:
raise ValueError("CPU should be at least equal to 1.")
hotplug_ram = ram_size - init_ram_size
if hotplug_ram < 0:
raise ValueError("RAM size cannot be less than the initial value %s MB" % init_ram_size)
if os.path.exists(status_path):
os.unlink(status_path)
qemu_wrapper = QemuQMPWrapper(socket_path)
qemu_wrapper.setVNCPassword(vnc_password)
if enable_device_hotplug == 'true':
write("Qemu Controller is still running...")
qemu_wrapper.updateDevice({
'device': 'cpu',
'amount': cpu_amount,
'model': cpu_model
})
qemu_wrapper.updateDevice({
'device': 'memory',
'mem': hotplug_ram,
'slot': slot_hotplug_size,
'nslot': 128,
'canreboot': 1
})
except Exception, e:
write(str(e))
raise
write("")
if __name__ == "__main__":
update()
#!{{ dash }}
if [ -f "{{ qemu_ready_path }}" ]; then
FILE="{{ qemu_ready_path }}"
# don't start checks too fast
sleep 2
if [ -f "$FILE" ]; then
if [ "$(cat $FILE)" = "" ]; then
echo "VM correctly started."
else
>&2 echo "Qemu Controller failed"
>&2 cat $FILE
exit 1
fi
else
log_file="{{ qemu_service_log_file }}"
>&2 echo "Qemu process is not correctly started."
......
......@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
listen_ip = '{{ parameter_dict.get("ipv4") }}'
mac_address = '{{ parameter_dict.get("mac-address") }}'
tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
smp_count = '{{ parameter_dict.get("smp-count") }}'
smp_options = '{{ parameter_dict.get("smp-options") }}'.strip()
numa_list = '{{ parameter_dict.get("numa") }}'.split()
ram_size = '{{ parameter_dict.get("ram-size") }}'
numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
ram_size = {{ parameter_dict.get("ram-size") }}
ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = '{{ parameter_dict.get("external-disk-size") }}'
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format") }}'
disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
......@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] and \
disk_cache == "directsync" else "threads"
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip()
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
logfile = '{{ parameter_dict.get("log-file") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
......@@ -255,15 +259,14 @@ if use_tap == 'true':
tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
smp = smp_count
if smp_options:
for option in smp_options.split(','):
key, val = option.split('=')
if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
smp += ',%s=%s' % (key, val)
if enable_device_hotplug != 'true':
smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%s' % (ram_size, ram_max_size)
else:
smp = '1,maxcpus=%s' % smp_max_count
ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name,
'-m', ram_size, '-vga', 'std',
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
'-vnc', '%s:1,ipv4,password' % listen_ip,
'-boot', 'order=cd,menu=on',
......
......@@ -17,7 +17,7 @@ parts +=
versions = versions
[versions]
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
[template-instance]
......
......@@ -116,8 +116,8 @@ mysqlclient = 1.3.12
persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
......
# TODO instance which runs this test periodically automatically and ingests results to ERP5
# NEO test instance: run neotest under Nexedi testing infrastructure
[buildout]
parts = runTestSuite
# std stuff for slapos instance
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
# software release we instantiate was supplied here
[software]
dir = {{ buildout['directory'] }}
bin = {{ buildout['bin-directory'] }}
# instance directories
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
bin = ${:home}/bin
neotest = ${:home}/neotest
# script to run the testsuite from inside <instance>/neotest/
# located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
[runTestSuite]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
# vvv appends "$@" to argv ^^^ without shell-escaping
parameters-extra = true
......@@ -2,6 +2,8 @@
. ${gowork:env.sh}
PATH="${coreutils:location}/bin:$PATH"
PATH="${util-linux:location}/bin:$PATH"
PATH="${ethtool:location}/sbin:$PATH"
PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH"
......
#!/bin/bash -e
# neotest's runTestSuite wraper so it could be run without any environment preset
. ${buildout:directory}/neotest-env.sh
exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"
......@@ -12,6 +12,8 @@ extends =
../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg
../../component/lmbench/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/util-linux/buildout.cfg
parts =
gowork
......@@ -19,6 +21,11 @@ parts =
ioping
ethtool
# neotest uses realpath & friends
coreutils
# ----//---- lsblk
util-linux
neoppod-develop
neoppod
wendelin.core-dev
......@@ -26,11 +33,12 @@ parts =
neotest-env.sh
neotest
neotest-runTestSuite
# for instance
slapos-deps-eggs
slapos-cookbook
# instance.cfg
instance.cfg
# go packages to install (+ automatically their dependencies)
......@@ -53,7 +61,7 @@ output = ${buildout:directory}/${:_buildout_section_name_}
[neotest-env.sh]
<= buildout-template
md5sum = 6e2203c7a86e8a3e8e56b8086115f5d4
md5sum = 595eebbbcb56b6a8464d48833cfae57b
[neotest]
<= buildout-template
......@@ -61,11 +69,25 @@ output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = fb3b4109128c1db1739ef5bb6abd1d94
[neotest-runTestSuite]
<= buildout-template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = 6a4281730b68cdba5c873817a6754428
# instance
[jinja2-template]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
rendered= ${buildout:directory}/${:_buildout_section_name_}
mode = 0644
context =
section buildout buildout
# instance (TODO)
[instance.cfg]
<= buildout-template
md5sum = 410e1b2d72829824b28cc0299adb472e
<= jinja2-template
md5sum = 7bd68ed8842cd25301bf04bcdcef88f6
# eggs:
......@@ -80,6 +102,8 @@ eggs =
wendelin.core
# for ZEO scripts (runzeo)
ZEO
# for nxd/runTestSuite
erp5.util
# wendelin.core: latest not yet released
......@@ -92,6 +116,7 @@ pyasn1 = 0.3.7
ZODB3 = 3.11.0
numpy = 1.13.3
zope.testing = 4.6.2
erp5.util = 0.4.50
# Required by:
# ZEO==4.3.1
......@@ -103,7 +128,3 @@ ZConfig = 3.2.0
# ZEO==4.3.1
# ZODB==4.4.5
zc.lockfile = 1.2.1
# Required by:
# slapos.toolbox==0.73
erp5.util = 0.4.49
......@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644
[versions]
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
dnspython = 1.15.0
PyRSS2Gen = 1.1
erp5.util = 0.4.50
passlib = 1.7.1
GitPython = 2.1.8
lockfile = 0.12.2
apache-libcloud = 2.2.1
feedparser = 5.2.1
atomize = 0.2.0
inotifyx = 0.2.2
gitdb2 = 2.0.3
pyasn1 = 0.4.2
smmap2 = 2.0.3
......@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -110,8 +110,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
# Required by:
......
......@@ -47,4 +47,4 @@ mode = 0644
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6
collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3
collective.recipe.template = 1.10
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
[template-runner]
filename = instance-runner.cfg
md5sum = 7df86928aeef0fbada832b0283b9a454
md5sum = 8b1caca52ab1307343ebada59f5a6c66
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 00ddcaa6e571505d1930612d12de8fde
md5sum = d1122988652127af5e252f1503eeacab
[template-resilient]
filename = instance-resilient.cfg.jinja2
......
......@@ -7,11 +7,13 @@ parts +=
nginx-launcher
certificate-authority
ca-nginx
logrotate-entry-nginx
gunicorn-launcher
gunicorn-graceful
publish-connection-information
slaprunner-promise
apache-httpd-promise
logrotate-entry-apache-httpd
slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key
runner-sshd-graceful
......@@ -30,6 +32,7 @@ parts +=
bash-profile
supervisord-wrapper
supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper
## Monitoring part
## Monitor for runner
......
......@@ -4,10 +4,12 @@ parts =
nginx-launcher
certificate-authority
ca-nginx
logrotate-entry-nginx
gunicorn-launcher
gunicorn-graceful
publish-connection-information
slaprunner-promise
logrotate-entry-apache-httpd
apache-httpd-promise
slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key
......@@ -27,6 +29,7 @@ parts =
bash-profile
supervisord-wrapper
supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper
{% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %}
slaprunner-frontend-promise
......@@ -347,6 +350,12 @@ mode = 700
context =
section param_nginx_frontend nginx-frontend
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = $${directory:log}/nginx.access.log $${directory:log}/nginx.error.log
post = kill -SIGUSR1 $(cat $${buildout:directory}/var/run/nginx.pid)
[httpd-parameters]
path_pid = $${directory:run}/httpd.pid
path_error_log = $${directory:log}/httpd-error.log
......@@ -387,6 +396,12 @@ wait-for-files =
$${ca-nginx:cert-file}
$${ca-nginx:key-file}
[logrotate-entry-apache-httpd]
<= logrotate-entry-base
name = apache
log = $${directory:log}/httpd-access.log $${directory:log}/httpd-error.log
post = test ! -s $${buildout:directory}/var/run/httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
[httpd-graceful-wrapper]
recipe = collective.recipe.template
input = inline:
......@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper
command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
wrapper-path = $${directory:services}/supervisord
[logrotate-entry-supervisord]
<= logrotate-entry-base
name = supervisord
log = $${directory:log}/slapproxy.log $${directory:log}/supervisord.log $${directory:log}/supervisord-errors.log
post = kill -SIGUSR2 $(cat $${buildout:directory}/var/run/supervisord.pid)
[supervisord-promise]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/supervisord
......
......@@ -15,8 +15,8 @@ gitdb = 0.6.4
gunicorn = 19.7.1
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
# Required by:
......
......@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11
plone.recipe.command = 1.1
slapos.recipe.build = 0.28
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout]
......
......@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640
[versions]
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
......@@ -94,8 +94,8 @@ caucase = 0.1.4
futures = 3.1.1
gitdb2 = 2.0.2
gunicorn = 19.7.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap2 = 2.0.3
# Required by:
......
......@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7
python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
# Required by:
# PasteScript==2.0
......
......@@ -63,6 +63,7 @@ extends =
../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg
../../component/postfix/buildout.cfg
../../component/zbarlight/buildout.cfg
../monitor/buildout.cfg
../../software/caucase/software.cfg
../../software/jupyter/software.cfg
......@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs}
${h5py:egg}
openpyxl
${statsmodels:egg}
${zbarlight:egg}
lock_file
astor
APacheDEX
......@@ -730,7 +732,7 @@ spyne = 2.12.14
suds = 0.4
facebook-sdk = 2.0.0
threadframe = 0.2
timerserver = 2.0.2
timerserver = 2.0.4
urlnorm = 1.1.4
uuid = 1.30
validictory = 1.1.0
......
......@@ -61,7 +61,7 @@ eggs =
# Monitor templates files
[monitor-httpd-conf]
<= monitor-template-base
md5sum = f2d6951670733de3b37c0ebe9eee343b
md5sum = b5f42503799e7e770afce4097d3b75ae
filename = monitor-httpd.conf.in
[monitor-template-wrapper]
......@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
[monitor-conf]
<= monitor-template-base
filename = monitor.conf.in
md5sum = fc65084dd1c2fe2487b58a003b576f61
md5sum = 888e2845d09bfaa59c25f56f5bcf76b1
[monitor-instance-info]
<= monitor-template-base
......@@ -131,6 +131,6 @@ depends =
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.73
slapos.toolbox = 0.74
pyasn1 = 0.3.7
......@@ -15,4 +15,4 @@
# not need these here).
[monitor2-template]
filename = instance-monitor.cfg.jinja2.in
md5sum = 5027f0b1abcc28ce3817cd70fb667a3b
md5sum = 03254b14a2ff242f7588a307d8c27f23
......@@ -42,6 +42,7 @@ services = ${directory:services}
services-conf = ${directory:etc}/monitor.conf.d
log = ${directory:log}/monitor
monitor-var = ${directory:var}/monitor
monitor-log = ${directory:monitor}/private/monitor-log
[ca-directory]
recipe = slapos.cookbook:mkdirectory
......@@ -88,14 +89,11 @@ promise-folder = ${directory:promises}
monitor-promise-folder = ${monitor-directory:promises}
promises-timeout-file = ${monitor-promise-timeout-file:file}
pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
randomsleep = {{ bin_directory }}/randomsleep
public-path-list =
private-path-list =
${directory:log}
#
monitor-url-list =
${monitor-instance-parameter:monitor-url-list}
private-path-list = ${directory:log}
monitor-url-list = ${monitor-instance-parameter:monitor-url-list}
parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
parameter-list =
......@@ -234,6 +232,12 @@ context =
raw dash_binary {{ dash_executable_location }}
command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[logrotate-entry-monitor-httpd]
<= logrotate-entry-base
name = monitor-apache
log = $${basedirectory:log}/monitor/monitor-httpd-error.log
post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1
[xnice-bin]
recipe = collective.recipe.template
input = inline:#!/bin/sh
......@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor-globalstate
frequency = * * * * *
command = ${monitor-globalstate-wrapper:wrapper-path}
command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path}
[monitor-configurator-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor-configurator
frequency = * * * * *
command = ${monitor-configurator-wrapper:wrapper-path}
command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path}
[monitor-httpd-promise]
recipe = slapos.cookbook:check_url_available
......@@ -407,6 +411,7 @@ depends =
${monitor-bootstrap-promise:file}
${promise-check-slapgrid:output}
${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
${logrotate-entry-monitor-httpd:name}
[monitor-publish]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
......
......@@ -2,6 +2,7 @@ PidFile "{{ parameter_dict.get('pid-file') }}"
StartServers 1
ServerLimit 1
MaxRequestWorkers 4
ThreadLimit 4
ThreadsPerChild 4
......
[monitor]
{% set monitor_url_list = parameter_dict.pop("monitor-url-list", "").strip() -%}
{% for key, value in parameter_dict.items() -%}
{% if key == "monitor-url-list" and monitor_base_urls is defined -%}
{{ key }} =
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endfor -%}
monitor-url-list =
{% if monitor_url_list -%}
{{ ' ' ~ monitor_url_list.replace("\n", "\n ") }}
{% else -%}
{% for key, value in monitor_base_urls.items() -%}
{{ ' ' ~ value }}
{% endfor -%}
{% else -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endif -%}
{% endfor -%}
......@@ -93,7 +93,7 @@ eggs =
[versions]
setuptools = 33.1.1
# Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos011
zc.buildout = 2.5.2+slapos012
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives
......@@ -121,14 +121,14 @@ netaddr = 0.7.19
pbr = 2.0.0
plone.recipe.command = 1.1
prettytable = 0.7.2
psutil = 5.2.0
psutil = 5.4.3
pyOpenSSL = 17.2.0
pyparsing = 2.2.0
pytz = 2016.10
requests = 2.13.0
six = 1.10.0
slapos.cookbook = 1.0.53
slapos.core = 1.4.3
slapos.core = 1.4.4
slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1
......@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7
paramiko = 2.1.3
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
Flask = 0.12
# Required by:
......@@ -160,7 +160,7 @@ ipaddress = 1.0.18
jsonschema = 2.6.0
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4
......@@ -173,15 +173,15 @@ packaging = 16.8
pycparser = 2.17
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
supervisor = 3.3.3
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
uritemplate = 3.0.0
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
zope.interface = 4.3.3
[networkcache]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment