Commit 7291d0fc authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate

parents 7427629c c53d0f43
......@@ -7,14 +7,15 @@ extends =
../bzip2/buildout.cfg
../zlib/buildout.cfg
../bzip2/buildout.cfg
../gperf/buildout.cfg
parts =
fontconfig
[fontconfig]
recipe = slapos.recipe.cmmi
url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2
md5sum = b5af5a423ee3b5cfc34846838963c058
url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
md5sum = 733f5e2371ca77b69707bd7b30cc2163
pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
# XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
configure-options =
......@@ -23,7 +24,7 @@ configure-options =
--enable-libxml2
--with-default-fonts=${fonts:location}
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends}
CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
......@@ -12,8 +12,8 @@ parts = haproxy
[haproxy]
recipe = slapos.recipe.cmmi
url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.9.tar.gz
md5sum = a2bbbdd45ffe18d99cdcf26aa992f92d
url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.1.tar.gz
md5sum = e42892d4b6ee33200fccaa1d81837e49
configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic".
......
......@@ -5,7 +5,8 @@ parts =
[jemalloc]
recipe = slapos.recipe.cmmi
url = http://www.canonware.com/download/jemalloc/jemalloc-3.6.0.tar.bz2
url = https://github.com/jemalloc/jemalloc/releases/download/${:version}/jemalloc-${:version}.tar.bz2
version = 3.6.0
md5sum = e76665b63a8fddf4c9f26d2fa67afdf2
configure-options =
--disable-static
......
......@@ -25,7 +25,7 @@ interpreter = keras-python
scripts = keras-python
[versions]
Keras = 2.0.8
tensorflow = 1.4.0rc1
Keras = 2.1.0
tensorflow = 1.4.0
h5py = 2.7.0rc2
Cython = 0.25.2
......@@ -12,7 +12,6 @@ extends =
../ncurses/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../pcre/buildout.cfg
../pkgconfig/buildout.cfg
../readline/buildout.cfg
../xz-utils/buildout.cfg
......@@ -27,8 +26,9 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-10.1.28/source/mariadb-10.1.28.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = 38acd5b44c56791701d80fddf088ef38
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.1.30
md5sum = c424fd12bdff388e3da1bdecf42626c9
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......@@ -45,7 +45,7 @@ configure-options =
-DWITH_ZLIB=system
-DWITH_READLINE=0
-DWITH_PIC=1
-DWITH_PCRE=system
-DWITH_PCRE=bundled
-DENABLE_DTRACE=0
-DWITH_EXTRA_CHARSETS=complex
-DWITH_EMBEDDED_SERVER=0
......@@ -56,22 +56,22 @@ configure-options =
-DWITH_INNODB_LZO=OFF
-DWITH_INNODB_SNAPPY=OFF
-DWITH_SAFEMALLOC=OFF
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-DWITHOUT_MROONGA_STORAGE_ENGINE=1
-DWITHOUT_DAEMON_EXAMPLE=1
-DPLUGIN_DAEMON_EXAMPLE=NO
-DPLUGIN_EXAMPLE=NO
-DPLUGIN_MROONGA=NO
-DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
extra_cflags =
extra_include_path =
extra_library_path =
environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
PATH=${patch:location}/bin:%(PATH)s
post-install =
mkdir -p ${:location}/include/wsrep &&
......@@ -108,6 +108,6 @@ pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment =
PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
CPPFLAGS=-I${groonga:location}/include/groonga
LDFLAGS=-L${groonga:location}/lib
PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig
# Do not extend any file that touch buildout:parts.
[mariadb]
url = https://downloads.mariadb.org/f/mariadb-10.2.9/source/mariadb-10.2.9.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = c59999bd182ddeb3db3d55250aecd8f8
version = 10.2.12
md5sum = c5bce588e3c53ebc417e37ecb0967aee
stable-patches =
configure-options +=
-DPLUGIN_DAEMON_EXAMPLE=NO
-DPLUGIN_EXAMPLE=NO
-DPLUGIN_MROONGA=NO
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
extra_cflags = -I${zstd:location}/include
......
......@@ -18,6 +18,7 @@ patch-options = -p0
patch-binary = ${patch:location}/bin/patch
patches =
http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
${:_profile_base_location_}/fix-gcc7-implicit-fallthrough-errors.patch
[nginx]
<= nginx-common
......
commit 8449f750e62cd229026e9df3bd023ec7e073a7d4
Author: Maxim Dounin <mdounin@mdounin.ru>
Date: Thu Apr 27 16:57:18 2017 +0300
Added missing "fall through" comments (ticket #1259).
Found by gcc7 (-Wimplicit-fallthrough).
diff --git src/core/ngx_murmurhash.c src/core/ngx_murmurhash.c
index c31e0e03..5ade658d 100644
--- src/core/ngx_murmurhash.c
+++ src/core/ngx_murmurhash.c
@@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t len)
switch (len) {
case 3:
h ^= data[2] << 16;
+ /* fall through */
case 2:
h ^= data[1] << 8;
+ /* fall through */
case 1:
h ^= data[0];
h *= 0x5bd1e995;
diff --git src/http/ngx_http_parse.c src/http/ngx_http_parse.c
index 36220fdc..e8e51563 100644
--- src/http/ngx_http_parse.c
+++ src/http/ngx_http_parse.c
@@ -1396,6 +1396,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1437,6 +1438,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1484,6 +1486,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
diff --git src/os/unix/ngx_process.c src/os/unix/ngx_process.c
index dd50b5ca..993c032a 100644
--- src/os/unix/ngx_process.c
+++ src/os/unix/ngx_process.c
@@ -413,6 +413,7 @@ ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
break;
}
ngx_debug_quit = 1;
+ /* fall through */
case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
ngx_quit = 1;
action = ", shutting down";
......@@ -32,8 +32,8 @@ setup-eggs =
${matplotlib:egg}
${pillow-python:egg}
patches =
${:_profile_base_location_}/ocropy.patch
patch-options = -p0
${:_profile_base_location_}/ocropy.patch#dd7a02e1e63ed9df68e3a539b3e919eb
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
environment = ocropy-env
find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz
......
diff --git ocrolib/__init__.py ocrolib/__init__.py
From 1bb1546b12b0c08b1b32b293207de2d58d43ff1c Mon Sep 17 00:00:00 2001
From: Francois Le Corre <francois.lecorre@nexedi.com>
Date: Thu, 6 Apr 2017 11:32:27 +0200
Subject: [PATCH] WIP
- New OCROPY_MODEL_PATH environment variable to specify the position of models
- Import for the first time a module that compiles C code from python,
allowing us to have the static library built before runtime and
preventing us from using gcc through zope.
---
ocrolib/__init__.py | 3 ++-
ocrolib/common.py | 7 ++++---
ocrolib/native.py | 1 +
setup.py | 15 ++++++++++++---
4 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/ocrolib/__init__.py b/ocrolib/__init__.py
index 1e0d627..81e85fb 100644
--- ocrolib/__init__.py
+++ ocrolib/__init__.py
--- a/ocrolib/__init__.py
+++ b/ocrolib/__init__.py
@@ -1,7 +1,7 @@
__all__ = [
"binnednn","cairoextras","common","components","dbtables",
......@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644
]
################################################################
@@ -9,5 +9,6 @@ __all__ = [
@@ -9,5 +9,6 @@
################################################################
import default
+from psegutils import *
from common import *
from default import traceback as trace
diff --git ocrolib/common.py ocrolib/common.py
diff --git a/ocrolib/common.py b/ocrolib/common.py
index 27c0f26..14f088f 100644
--- ocrolib/common.py
+++ ocrolib/common.py
@@ -14,6 +14,7 @@ import unicodedata
--- a/ocrolib/common.py
+++ b/ocrolib/common.py
@@ -14,6 +14,7 @@
import inspect
import glob
import cPickle
......@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644
def load_object(fname,zip=0,nofind=0,verbose=0):
"""Loads an object from disk. By default, this handles zipped files
and searches in the usual places for OCRopus. It also handles some
@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0):
@@ -439,8 +441,7 @@ class names that have changed."""
if zip==0 and fname.endswith(".gz"):
zip = 1
if zip>0:
......@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644
possible_prefixes.append("/usr/local/share/ocropus")
diff --git ocrolib/native.py ocrolib/native.py
diff --git a/ocrolib/native.py b/ocrolib/native.py
index b7a207f..240450b 100644
--- ocrolib/native.py
+++ ocrolib/native.py
--- a/ocrolib/native.py
+++ b/ocrolib/native.py
@@ -44,6 +44,7 @@ class CompileError(Exception):
def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
......@@ -69,22 +85,22 @@ index b7a207f..240450b 100644
if not os.path.exists(prefix):
os.mkdir(prefix)
m = hashlib.md5()
diff --git setup.py setup.py
index 2ec5832..6697b12 100644
--- setup.py
+++ setup.py
@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\
diff --git a/setup.py b/setup.py
index 2ec5832..0ad4d85 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,9 @@
from distutils.core import setup #, Extension, Command
#from distutils.command.install_data import install_data
-if not os.path.exists("models/en-default.pyrnn.gz"):
+models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \
+ [c for c in glob.glob("models/*pyrnn.gz")]
+models = os.environ.get('OCROPY_MODEL_PATH')
+models = models.split(':') if models else glob.glob("models/*pyrnn.gz")
+if not models:
print()
print("You should download the default model 'en-default.pyrnn.gz'")
print("and put it into ./models.")
@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"):
@@ -18,16 +20,23 @@
print("Check https://github.com/tmbdev/ocropy for the location")
print("of model files.")
print()
......@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644
+ data_files= [('share/ocropus', models), ("", ["LICENSE"])],
scripts = scripts,
)
--
2.14.1
......@@ -28,3 +28,12 @@ configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--exec-prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty
[openssh-output]
# Shared binary location to ease migration
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = ${coreutils-output:test} -x ${:ssh} -a -x ${:keygen}
ssh = ${openssh:location}/bin/ssh
keygen = ${openssh:location}/bin/ssh-keygen
\ No newline at end of file
......@@ -16,8 +16,8 @@ parts =
[openssl]
recipe = slapos.recipe.cmmi
url = https://www.openssl.org/source/openssl-1.0.2m.tar.gz
md5sum = 10e9e37f492094b9ef296f68f24a7666
url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz
md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4
location = ${buildout:parts-directory}/${:_buildout_section_name_}
# 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
......@@ -22,6 +22,13 @@ rpath =
# and pin the egg in the [versions] section of the stack or SR.
find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz
[rdiff-backup-build-1.3.4]
<= rdiff-backup-build
# use our own version
find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
patches =
${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
[rdiff-backup]
# Scripts only generation part for rdiff-backup
recipe = zc.recipe.egg
......
Patch by Roman Tereshonkov and Kari Hautio for rdiff-backup <= 1.2.8 to avoid a build failure with
librsync >= 1.0.0 (which is a security bugfix release). The discussion and solution finding can be
found at https://bugs.launchpad.net/duplicity/+bug/1416344 (for duplicity).
--- rdiff-backup-1.3.4/rdiff_backup/_librsyncmodule.c 2009-03-16 15:36:21.000000000 +0100
+++ rdiff-backup-1.3.4/rdiff_backup/_librsyncmodule.c.librsync-1.0.0 2015-03-02 00:54:24.000000000 +0100
@@ -59,8 +59,13 @@
if (sm == NULL) return NULL;
sm->x_attr = NULL;
+#ifdef RS_DEFAULT_STRONG_LEN
sm->sig_job = rs_sig_begin((size_t)blocklen,
(size_t)RS_DEFAULT_STRONG_LEN);
+#else
+ sm->sig_job = rs_sig_begin((size_t)blocklen,
+ (size_t)8, RS_MD4_SIG_MAGIC);
+#endif
return (PyObject*)sm;
}
......@@ -56,14 +56,16 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location}
egg = tensorflow-tensorboard
bazel-bin = ${bazel:location}/bin
java_home_bin = ${bazel:java_home}/bin
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script =
os.makedirs(location)
workdir = self.options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0rc1-py2.7.egg'
egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'],
self.options['java_home_bin'],
os.environ['PATH']]),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
}
......
......@@ -27,7 +27,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/tensorflow/tensorflow
tag = r1.4
tag = v1.4.0
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
......@@ -110,7 +110,7 @@ egg = tensorflow
script =
os.makedirs(location)
workdir = self.options['tensorflow-repository-path']
egg_name = 'tensorflow-1.4.0rc1-py2.7-linux-x86_64.egg'
egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
......
......@@ -55,11 +55,12 @@ setup(name=name,
packages=find_packages(),
include_package_data=True,
install_requires=[
'enum34', # for inotify-simple
'jsonschema',
'hexagonit.recipe.download',
'netaddr', # to manipulate on IP addresses
'setuptools', # namespaces
'inotifyx', # XXX use pyinotify instead
'inotify_simple',
'lock_file', #another lockfile implementation for multiprocess
'slapos.core', # uses internally
'zc.buildout', # plays with buildout
......
......@@ -85,7 +85,7 @@ class Recipe(GenericBaseRecipe):
mysql_connection_string_list),
] + common_list, **common_dict)]))
path_list.append(self.createPythonScript(self.options['run-test-suite'],
__name__ + '.test.runUnitTest', [dict(
__name__ + '.test.runTestSuite', [dict(
call_list=[self.options['run-test-suite-binary'],
'--db_list', ','.join(mysql_connection_string_list),
] + common_list, **common_dict)]))
......
......@@ -34,7 +34,10 @@ def runTestSuite(args):
if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
try:
env['PATH'] = d['prepend_path'] + ':' + env['PATH']
except KeyError:
env['PATH'] = d['prepend_path']
if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home']
......@@ -57,29 +60,4 @@ def runTestSuite(args):
argument_list.append(env)
os.execle(executable_filepath, *argument_list)
def runUnitTest(args):
env = os.environ.copy()
d = args[0]
if 'openssl_binary' in d:
env['OPENSSL_BINARY'] = d['openssl_binary']
if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home']
# Deal with Shebang size limitation
executable_filepath = d['call_list'][0]
file_object = open(executable_filepath, 'r')
line = file_object.readline()
file_object.close()
argument_list = []
if line[:2] == '#!':
executable_filepath = line[2:].strip()
argument_list.append(executable_filepath)
argument_list.extend(d['call_list'])
argument_list.extend(sys.argv[1:])
argument_list.append(env)
os.execle(executable_filepath, *argument_list)
runUnitTest = runTestSuite
......@@ -3,47 +3,31 @@ import os
import signal
import subprocess
import time
import inotifyx
from collections import defaultdict
from inotify_simple import INotify, flags
def _wait_files_creation(file_list):
# Etablish a list of directory and subfiles
directories = dict()
for dirname, filename in [os.path.split(f) for f in file_list]:
directories.setdefault(dirname, dict())
directories[dirname][filename] = False
# Establish a list of directory and subfiles.
# and test existence before watching, so that we don't miss an event.
directories = defaultdict(dict)
for f in file_list:
dirname, filename = os.path.split(f)
directories[dirname][filename] = os.path.lexists(f)
def all_files_exists():
return all([all(files.values()) for files in directories.values()])
return all(all(files.itervalues()) for files in directories.itervalues())
fd = inotifyx.init()
try:
# Watch every directories where the file are
watchdescriptors = dict()
for dirname in directories.keys():
wd = inotifyx.add_watch(fd,
dirname,
inotifyx.IN_CREATE | inotifyx.IN_DELETE | inotifyx.IN_MOVE)
watchdescriptors[wd] = dirname
# Set to True the file wich exists
for dirname, filename in [os.path.split(f) for f in file_list]:
directories[dirname][filename] = os.path.exists(os.path.join(dirname,
filename))
# Let's wait for every file creation
while not all_files_exists():
events_list = inotifyx.get_events(fd)
for event in events_list:
dirname = watchdescriptors[event.wd]
if event.name in directories[dirname]:
# One of watched file was created or deleted
if event.mask & inotifyx.IN_DELETE:
directories[dirname][event.name] = False
else:
directories[dirname][event.name] = True
with INotify() as inotify:
watchdescriptors = {inotify.add_watch(dirname,
flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
): dirname
for dirname in directories}
finally:
os.close(fd)
while not all_files_exists():
for event in inotify.read():
directory = directories[watchdescriptors[event.wd]]
if event.name in directory:
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
def execute(args):
"""Portable execution with process replacement"""
......@@ -83,8 +67,8 @@ def generic_exec(args):
os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
def sig_handler(signal, frame):
print 'Received signal %r, killing children and exiting' % signal
def sig_handler(sig, frame):
print 'Received signal %r, killing children and exiting' % sig
if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM)
......@@ -97,6 +81,7 @@ signal.signal(signal.SIGTERM, sig_handler)
def execute_with_signal_translation(args):
"""Run process as children and translate from SIGTERM to another signal"""
global child_pg
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid
try:
......
......@@ -25,8 +25,7 @@
#
##############################################################################
import os
import inotifyx
from inotify_simple import INotify, flags
def subfiles(directory):
"""Return the list of subfiles of a directory, and wait for the newly created
......@@ -34,18 +33,12 @@ def subfiles(directory):
CAUTION : *DONT TRY TO CONVERT THE RESULT OF THIS FUNCTION INTO A LIST !
ALWAYS ITERATE OVER IT !!!*"""
watchfd = inotifyx.init()
inotifyx.add_watch(watchfd, directory, inotifyx.IN_CREATE)
try:
subfiles = set(os.listdir(directory))
subfiles |= set([file_.name for file_ in inotifyx.get_events(watchfd, 0)])
with INotify() as inotify:
inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO)
names = os.listdir(directory)
while True:
for file_ in subfiles:
yield os.path.join(directory, file_)
subfiles = [file_.name for file_ in inotifyx.get_events(watchfd)]
finally:
os.close(watchfd)
for name in names:
yield os.path.join(directory, name)
names = (event.name for event in inotify.read())
......@@ -92,6 +92,8 @@ class Storage(NeoBaseRecipe):
engine = self.options.get('engine')
if engine: # old versions of NEO don't support -e
r += '-e', engine
if self.options.get('dedup'):
r.append('--dedup')
if self.options.get('disable-drop-partitions'):
r.append('--disable-drop-partitions')
return r
......
import os, shutil, tempfile, threading, unittest
from slapos.recipe.librecipe import execute, inotify
class TestInotify(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmp)
def test_subfiles(self):
p = lambda x: os.path.join(self.tmp, x)
def create(name, text):
a = open(p(name), 'w')
a.write(text)
a.flush()
return a
def check(name, text):
path = next(notified)
self.assertEqual(path, p(name))
with open(path) as f:
self.assertEqual(f.read(), text)
a = create('first', 'blah')
a.write('...')
notified = inotify.subfiles(self.tmp)
check('first', 'blah')
os.link(p(a.name), p('a hard link')) # ignored
b = create('other', 'hello')
b.close()
check('other', 'hello')
c = create('last', '!!!')
a.close()
check('first', 'blah...')
os.rename(p(a.name), p(b.name))
check('other', 'blah...')
c.close()
check('last', '!!!')
def test_wait_files_creation(self):
file_list = (
'foo',
'bar',
'hello/world',
'hello/world!',
'a/b/c',
)
create = lambda x: open(x, 'w').close()
p = lambda x: os.path.join(self.tmp, x)
P = lambda x: p(file_list[x])
create(P(1))
os.mkdir(p('hello'))
os.makedirs(p('a/b'))
t = threading.Thread(target=execute._wait_files_creation,
args=(map(p, file_list),))
t.daemon = True
t.start()
def check():
t.join(.2)
self.assertTrue(t.is_alive())
check()
for x in P(3), p('a/b/d'), P(0):
create(x)
check()
os.rename(P(3), P(2))
os.rename(p('a/b/d'), P(4))
check()
os.remove(P(1))
for x in P(3), P(1):
create(x)
t.join(10)
self.assertFalse(t.is_alive())
......@@ -50,7 +50,7 @@ gitdb = 0.6.4
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
smmap = 0.9.0
......
......@@ -10,7 +10,7 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
smmap = 0.9.0
numpy = 1.11.2
......
......@@ -27,6 +27,7 @@ statistic = $${:srv}/statistic
backupscript = $${:etc}/backup
www = $${:srv}/www
home = $${:etc}/home
promises = $${:etc}/promise
ssl = $${:etc}/ssl
ssh = $${:home}/.ssh
......@@ -49,12 +50,13 @@ logfile = $${directory:log}/crond.log
{% set frequency = slave_instance.get('frequency', '') -%}
{% set hostname = slave_instance.get('hostname', '') -%}
{% set connection = slave_instance.get('connection', '') -%}
{% set connection_port = slave_instance.get('connection_port', '22') -%}
{% set include = slave_instance.get('include', '') -%}
{% set include_string = "' --include='".join(include.split(' ')) -%}
{% set exclude = slave_instance.get('exclude', '') -%}
{% set exclude_string = '' -%}
{% set sudo = slave_instance.get('sudo', 'False') -%}
{% set remote_schema = 'rdiff-backup --server --restrict-read-only / -- "$@"' -%}
{% set remote_schema = slave_instance.get('remote_rdiff_path', 'rdiff-backup') + ' --server --restrict-read-only / -- "$@"' -%}
{% if (exclude != '') -%}
{% set exclude_string = "' --exclude='".join(exclude.split(' ')) -%}
......@@ -73,23 +75,19 @@ directory = $${directory:backup}/$${:_buildout_section_name_}
[{{ slave_reference }}-backup-private_key]
recipe = plone.recipe.command
stop-on-error = false
command = ${dropbear-output:keygen} -t $${:type} -s 2048 -f $${:key}
stop-on-error = true
command = ${coreutils-output:rm} -f $${:key} $${:public_key} && ${openssh-output:keygen} -t $${:type} -b 2048 -f $${:key} -q -N ""
key = $${directory:ssh}/$${:_buildout_section_name_}
public_key = $${:key}.pub
location = $${:public_key}
type = rsa
[{{ slave_reference }}-backup-public_key]
recipe = plone.recipe.command
stop-on-error = true
command = ${coreutils-output:rm} -f $${:key} && ${dropbear-output:keygen} -y -f {{ '$${' ~ slave_reference }}-backup-private_key:key} | ${grep-output:grep} {{ '$${' ~ slave_reference }}-backup-private_key:type} > $${:key}
key = {{ '$${' ~ slave_reference }}-backup-private_key:key}.pub
location = $${:key}
# Insert as a beginning part, to ensure that all public keys are generated before trying to publish. This will reduce the number of slapgrid-cp run.
{% do part_list.insert(0, "%s-backup-public_key" % slave_reference) -%}
{% do part_list.insert(0, "%s-backup-private_key" % slave_reference) -%}
[{{ slave_reference }}-backup-read-public_key]
recipe = slapos.cookbook:readline
storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
storage-path = {{ '$${' ~ slave_reference }}-backup-private_key:public_key}
# Publish slave {{ slave_reference }} information
[{{ slave_reference }}-backup-publish]
......@@ -107,6 +105,7 @@ mode = 0700
datadirectory = {{ '$${' ~ slave_reference }}-backup-directory:directory}
sshkey = {{ '$${' ~ slave_reference }}-backup-private_key:key}
connection = {{ connection }}
connection_port = {{ connection_port }}
hostname = {{ hostname }}
include = {{ include_string }}
exclude_string = {{ exclude_string }}
......@@ -169,6 +168,12 @@ mode = 0700
virtual-depends =
$${nginx-configuration:ip}
[nginx-listen-promise]
recipe = slapos.cookbook:check_port_listening
hostname = $${nginx-configuration:ip}
port = $${nginx-configuration:port}
path = $${directory:promises}/nginx_listen
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
......@@ -187,6 +192,7 @@ ssl_crt = $${directory:ssl}/nginx.crt
parts =
dcron-service
nginx-service
nginx-listen-promise
activate-crontab-file
publish-global-rss
{% for part in part_list -%}
......
......@@ -10,7 +10,7 @@ extends =
# ../../component/git/buildout.cfg
# ../../component/subversion/buildout.cfg
../../component/rsync/buildout.cfg
../../component/dropbear/buildout.cfg
../../component/openssh/buildout.cfg
../../component/grep/buildout.cfg
../../component/findutils/buildout.cfg
# ../../stack/flask.cfg
......@@ -67,7 +67,7 @@ mode = 0644
[template-backup-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-backup-script.sh.in
md5sum = 47b20031db3b575651d8515d5add23e6
md5sum = fa79e0307e12e2f5b1f2adbd261995fc
output = ${buildout:directory}/template-backup-script.sh.in
mode = 0644
......@@ -88,7 +88,7 @@ mode = 0644
[status2rss]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/status2rss.py
md5sum = 0cd1cf97b199dd18fc0168c7281890ea
md5sum = a023694817975e73998fb9187a6015d6
output = ${buildout:directory}/status2rss.py
mode = 0644
......@@ -105,7 +105,7 @@ mode = 0644
[template-pullrdiffbackup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
md5sum = 061b98d001b501c9e1beb424e8802d3d
md5sum = a2fb7b0cdd944be99da4122eb6f07749
output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644
......@@ -116,8 +116,14 @@ md5sum = 42021b325159dff29e4bd4e33b8ff2f3
output = ${buildout:directory}/template.cfg
mode = 0644
[rdiff-backup]
eggs =
${rdiff-backup-build-1.3.4:egg}
[versions]
rdiff-backup = 1.0.5+SlapOSPatched001
# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
# but '1.3.4nxd2-SlapOSPatched001'.
rdiff-backup = 1.3.4nxd2-SlapOSPatched001
gunicorn = 19.1.1
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.2
......
......@@ -29,9 +29,8 @@ while 1:
title = desc,
description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc,
open(statistic).read())).split("\n")),
link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)), isPermaLink=0)
)
items.append(rss_item)
......
......@@ -18,7 +18,7 @@ ${rdiff-backup-output:rdiff-backup} \
$${:exclude_string} \
--include='$${:include}' \
--exclude='**' \
--remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
--remote-schema '${openssh-output:ssh} -6 -q -T -y -o "StrictHostKeyChecking no" -i $${:sshkey} -p $${:connection_port} %s $${:remote_schema}' \
$${:connection}::/ ./
RESULT=$?
......
......@@ -77,7 +77,7 @@ def main():
test_title = args.test_suite_title or args.test_suite
if args.master_url:
tool = taskdistribution.TaskDistributionTool(args.master_url)
tool = taskdistribution.TaskDistributor(args.master_url)
test_result = tool.createTestResult(args.revision,
list(test_dict),
args.test_node_title,
......
......@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
erp5.util = 0.4.49
plone.recipe.command = 1.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
......@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies
cp.recipe.cmd = 0.5
plone.recipe.command = 1.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
zope.exceptions = 4.0.7
zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0
......
......@@ -3,12 +3,8 @@ Available ``software-type`` values
- ``default``
Recommended for production use.
- ``create-erp5-site``
Automated creation of ERP5Site instance, for easy deployment.
Usage in production discouraged due to the increased risk of data loss.
Recommended for developemnt and production use. Automatic creation of
erp5-site.
Notes
=====
......
......@@ -9,13 +9,6 @@
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 0
},
"create-erp5-site": {
"title": "Create ERP5 Site",
"description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 1
}
}
}
......@@ -58,8 +58,8 @@ mode = 0644
[versions]
PyXML = 0.8.5
erp5.util = 0.4.49
slapos.recipe.template = 4.1
erp5.util = 0.4.50
slapos.recipe.template = 4.2
ipython = 5.3.0
apache-libcloud = 2.1.0
gitdb2 = 2.0.2
......
......@@ -158,10 +158,10 @@ def main():
access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
tool = taskdistribution.TaskDistributionTool(
distributor = taskdistribution.TaskDistributor(
args.master_url,
logger=logger)
test_result = tool.createTestResult(
test_result = distributor.createTestResult(
revision, suite.getTestList(), args.test_node_title,
suite.allow_restart, test_suite_title, args.project_title)
if test_result is None:
......
......@@ -71,4 +71,4 @@ output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
erp5.util = 0.4.49
erp5.util = 0.4.50
......@@ -5,6 +5,7 @@ extends =
parts =
instance-template
slapos-cookbook
[instance-template]
recipe = slapos.recipe.template
......@@ -14,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644
[versions]
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
......@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
z3c.recipe.scripts = 1.0.1
......@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4
pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.40.4
smmap = 0.8.2
plone.recipe.command = 1.1
......
......@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
......@@ -91,8 +91,8 @@ def main():
if args.target == 'firefox':
firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
firefox_capabilities['binary'] = '${firefox:location}/firefox-slapos'
browser = webdriver.Firefox(capabilities=firefox_capabilities,
firefox_binary='${firefox:location}/firefox-slapos',
executable_path='${firefox:location}/geckodriver')
elif args.target in ['iOS', 'Android']:
# parameters for mobile emulators have different names then parameters for
......@@ -169,7 +169,7 @@ def main():
browser.quit()
is_browser_running = False
tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url)
tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
test_result = tool.createTestResult(revision = revision,
test_name_list = test_line_dict.keys(),
node_title = args.test_node_title,
......
......@@ -107,11 +107,11 @@ mode = 0644
[template-runTestSuite]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/runTestSuite.in
md5sum = fcf15b2a90340e0afe8f8b9921a4ffae
md5sum = ff66d13f73982e8257eb5535cdb541c7
output = ${buildout:directory}/runTestSuite.in
mode = 0644
[versions]
erp5.util = 0.4.49
slapos.recipe.template = 4.1
selenium = 2.53.1
erp5.util = 0.4.50
slapos.recipe.template = 4.2
selenium = 3.8.0
......@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1
seaborn = 0.7.1
simplegeneric = 0.8.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
statsmodels = 0.8.0
terminado = 0.6
tornado = 4.4.2
......
......@@ -11,7 +11,7 @@ apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
pycurl = 7.43.0
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
smmap = 0.9.0
# websockify = 0.8.0
......
......@@ -17,7 +17,7 @@ parts +=
versions = versions
[versions]
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
[template-instance]
......
......@@ -70,6 +70,11 @@
"default": 1,
"type": "integer"
},
"data-deduplication": {
"description": "Set the --dedup option for storage nodes.",
"default": false,
"type": "boolean"
},
"disable-drop-partitions": {
"description": "Set the --disable-drop-partitions option for storage nodes.",
"default": false,
......
......@@ -87,6 +87,7 @@ masters = ${publish:masters}
database-adapter = MySQL
wait-database = -1
engine = {{ slapparameter_dict.get('engine', '') }}
dedup = {{ dumps(bool(slapparameter_dict.get('data-deduplication'))) }}
disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-partitions'))) }}
{% for i in range(slapparameter_dict.get('storage-count', 1)) -%}
......
......@@ -98,7 +98,7 @@ md5sum = 1fee10f02c2fa2a581e21878ca0fd704
[instance-neo-storage-mysql]
<= download-base-neo
md5sum = 67d623d631c2f99e33bcabc79fc9cccf
md5sum = 366e51c0dbd85e511a31e403b8704735
[template-neo-my-cnf]
<= download-base-neo
......@@ -116,7 +116,7 @@ mysqlclient = 1.3.12
persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
smmap2 = 2.0.1
transaction = 1.7.0
......
......@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644
[versions]
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
......@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
......@@ -110,7 +110,7 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
smmap = 0.9.0
......
......@@ -47,4 +47,4 @@ mode = 0644
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
......@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6
collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
......@@ -15,7 +15,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 251c1fcec1817fe70ec0cf7da69ccf3a
md5sum = 4a77ee4a6367fee27552f8bfe9d87aab
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -143,7 +143,7 @@ return =
zope-address-list
hosts-dict
monitor-base-url
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc slapos_configurator' -%}
{% set bt5_default_list = 'erp5_full_text_myisam_catalog slapos_configurator' -%}
{% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
{% endif -%}
......
......@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3
collective.recipe.template = 1.10
plone.recipe.command = 1.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
......@@ -152,6 +152,7 @@ eggs =
gunicorn==19.7.1
futures
${slapos-cookbook:eggs}
slapos.core # listed explicitly for scripts generation
[extra-eggs]
recipe = zc.recipe.egg
......
......@@ -15,7 +15,7 @@ gitdb = 0.6.4
gunicorn = 19.7.1
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
smmap = 0.9.0
......
......@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11
plone.recipe.command = 1.1
slapos.recipe.build = 0.28
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout]
......
......@@ -51,7 +51,6 @@ MarkupSafe = 0.18
Werkzeug = 0.8.3
buildout-versions = 1.7
hexagonit.recipe.cmmi = 2.0
inotifyx = 0.2.0-1
lxml = 3.2.1
meld3 = 0.6.10
netaddr = 0.7.10
......
......@@ -44,7 +44,6 @@ hexagonit.recipe.download = 1.6nxd002
# Required by:
# slapos.cookbook==0.73.1
inotifyx = 0.2.0
# Required by:
# slapos.cookbook==0.73.1
......
......@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640
[versions]
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
[buildout]
extends =
../wendelin/software.cfg
../../component/mariadb/mariarocks.cfg
[local-bt5-repository]
list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
# ERP5: id-store-interval must be big enough to avoid conflicts (e.g. 1000)
[buildout]
extends = test-fluentd-common.cfg
......@@ -21,7 +23,6 @@ recipe = slapos.recipe.build:download
[template-zope-patched]
<= template-fluentd
base = ${template-zope-base:target}
method = unpack
tags =
{%- if slapparameter_dict['family'] == 'fluentd' %}
{%- for i, zope in enumerate(publish_list) %}
......@@ -32,7 +33,7 @@ tags =
[feeder]
feeder =
#
# Same algorithm as ERP5Site_simulateFluentdIngestion
import os, struct
from random import lognormvariate
pack = struct.Struct('!d').pack
......
[buildout]
extends =
../../component/gnupg/buildout.cfg
../fluentd/software.cfg
../wendelin/software.cfg
../../component/mariadb/mariarocks.cfg
[local-bt5-repository]
list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
test-common.cfg
[template-erp5]
recipe =
......@@ -32,6 +17,7 @@ base = ${template-erp5-base:target}
[template-fluentd]
<= patch-template
method = unpack
extra =
{%- set tags = {} %}
${:tags}
......
# ERP5: id-store-interval must be big enough to avoid conflicts (e.g. 1000)
# XXX: Because supervisord can't handle too many processes
# ("too many open files to spawn" errors), these SR
# should instantiate the source processes differently.
[buildout]
extends = test-fluentd-common.cfg
[template-erp5-patched]
extra =
[request-balancer]
config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = $${publish-early:inituser-password}
[template-balancer]
recipe =
target = ${template-balancer-patched:location}
[template-balancer-base]
<= template-balancer
recipe = slapos.recipe.build:download
[template-balancer-patched]
<= template-fluentd
base = ${template-balancer-base:target}
tags =
{%- set port, backend_list = haproxy_dict['fluentd'] %}
{%- for i in range(100*len(backend_list)) %}
{%- do tags.__setitem__('wendelin_tag_' ~ i, ipv4 ~ ':' ~ port) %}
{%- endfor %}
[feeder]
feeder =
#
import collections, random, struct
pack = struct.Struct('!d').pack
data = collections.deque(
''.join(chr(int(random.gauss(0, .68)) % 256) for _ in xrange(2500))
# With a period greater than 64kiB (zlib dictionary size),
# we avoid extra compression due to repetition.
for _ in xrange(30))
interval = 60 # XXX: same as fluentd flush internal
time.sleep(interval * random.random())
while True:
emit('', pack(time.time()) + data[0])
data.rotate()
time.sleep(interval)
# NEO: data deduplication must be enabled
[buildout]
extends = test-fluentd-common.cfg
extends =
../../component/gnupg/buildout.cfg
test-fluentd-common.cfg
[template-erp5-patched]
extra =
......
# The 'start_ingest' command causes a zope to fill NEO as fast as possible,
# as if fluentd pushed data.
#
# Use sigma > 0 to have oids of variable size inside NEO.
# To know average compression ratio:
# x=test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py
# $x 10 1
# 0.434851958247
# 2155 - 65536 (99th percentile)
# $x 8.787 0
# 0.100036621094
# 6556
[buildout]
extends = test-common.cfg
parts += start_ingest
[start_ingest]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
template =
inline:#!${buildout:executable}
import argparse, base64, httplib, sys
parser = argparse.ArgumentParser()
_ = parser.add_argument
_('--site-id', default='erp5')
_('hostport', metavar='host[:port]', help='Zope address')
_('password', help="'zope' user password")
_('reference', help='Data Stream reference')
_('mu', type=float)
_('sigma', type=float)
_('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
args = parser.parse_args()
qs = []
for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
v = getattr(args, k)
if v is not None:
t = type(v)
qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
c = httplib.HTTPConnection(args.hostport)
c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
% (args.site_id, '&'.join(qs)))
c.putheader('Authorization',
'Basic ' + base64.b64encode('zope:'+args.password))
c.endheaders()
#!/usr/bin/python
from __future__ import division, print_function
import os, struct
from random import lognormvariate
bigfile_chunk_size = 65536
def simulateFluentdIngestion(self, reference, mu, sigma,
chunks_per_transaction=128):
from time import time
import transaction
note = (self['portal_ingestion_policies']['scalability_test_unpack'].getPath()
+ '/ingest')
module = self['data_stream_module']
try:
data_stream = module[reference]
except KeyError:
data_stream = module.newContent(reference, 'Data Stream')
transaction.commit()
pack = struct.Struct('!d').pack
data = os.urandom(bigfile_chunk_size - 8)
while 1:
txn = transaction.begin()
data_stream.appendData(''.join(
(pack(time()) + data[:int(lognormvariate(mu, sigma))]
).ljust(bigfile_chunk_size, '\0')
for _ in xrange(chunks_per_transaction)))
txn.note(note)
txn.commit()
if __name__ == '__main__':
import sys
mu, sigma = map(float, sys.argv[1:3])
if sigma:
try:
n = int(sys.argv[3])
except IndexError:
n = 1000000
else:
n = 1
x = sorted(min(int(lognormvariate(mu, sigma)), bigfile_chunk_size - 8)
for _ in xrange(n))
print((8 * n + sum(x)) / (bigfile_chunk_size * n))
if n == 1:
print(x[0] + 8)
else:
n //= 100
if n:
print(8 + x[n], '-', 8 + x[-n-1], '(99th percentile)')
else:
print(8 + x[0], '-', 8 + x[-1])
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 8, 46: Redefining name \'mu\' from outer scope (line 35) (redefined-outer-name)</string>
<string>W: 8, 50: Redefining name \'sigma\' from outer scope (line 35) (redefined-outer-name)</string>
</tuple>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>simulateFluentdIngestion</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>ScalabilityFluentd</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_simulateFluentdIngestion</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
extension.erp5.ScalabilityFluentd
\ No newline at end of file
portal_ingestion_policies/scalability_test_*
portal_skins/custom/DataStreamModule_getTotalSize
portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file
......@@ -10,6 +10,8 @@ tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} {
log {{caddy_configuration_dict['access_log']}}
errors {{caddy_configuration_dict['error_log']}}
root {{ directory_dict['public_html'] }}
{% if parameter_dict['enable-basic-auth'] == 'true' -%}
......@@ -18,6 +20,8 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
}
bind {{caddy_configuration_dict['ipv6']}}
{%- endif %}
[{{caddy_configuration_dict['ipv6']}}]:{{parameter_dict['port-ipv6']}} {
......@@ -30,22 +34,30 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
log {{caddy_configuration_dict['access_log']}}
errors {{caddy_configuration_dict['error_log']}}
root {{ directory_dict['public_html'] }}
{% if parameter_dict['enable-basic-auth'] == 'true' -%}
basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
{%- endif %}
bind {{caddy_configuration_dict['ipv6']}}
}
[{{caddy_configuration_dict['local_ip']}}]:{{parameter_dict['port-ipv4']}} {
log {{caddy_configuration_dict['access_log']}}
errors {{caddy_configuration_dict['error_log']}}
root {{ directory_dict['public_html'] }}
{% if parameter_dict['enable-basic-auth'] == 'true' -%}
basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
{%- endif %}
bind {{caddy_configuration_dict['local_ip']}}
}
\ No newline at end of file
......@@ -27,7 +27,7 @@ mode = 0644
[template-caddyfile]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/Caddyfile.in
#md5sum = bb3b314ebeb58e9875d547a053a9f268
md5sum = 88c4c33e374ea3f61cdd36b2816d24ba
filename = Caddyfile.in
location = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 0644
......
......@@ -94,7 +94,7 @@ caucase = 0.1.4
futures = 3.1.1
gitdb2 = 2.0.2
gunicorn = 19.7.1
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
smmap2 = 2.0.3
......
......@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7
python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.1
slapos.recipe.template = 4.2
# Required by:
# PasteScript==2.0
......
......@@ -736,7 +736,7 @@ uuid = 1.30
validictory = 1.1.0
xfw = 0.10
xupdate-processor = 0.4
selenium = 2.53.1
selenium = 3.8.0
# Required by:
# Products.CMFCore==2.2.10
......
......@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d
[mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in
md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
[mariadb-start-clone-from-backup]
filename = instance-mariadb-start-clone-from-backup.sh.in
......@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 8ea5033142f450a2e90431817771cb44
md5sum = da7c36ecb490b67360d2afda94b41bff
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 2465af81147af322056cee9f6c7de14f
md5sum = 02ed5d9b74c70789004d01dd2ecde7b1
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -91,7 +91,7 @@ md5sum = fd7e8c507cef1950e6c0347ce2a01021
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = f64c568f1365eb1164f12f48fede9a99
md5sum = a71ad387eab681b9020e271cba2c7a79
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
......@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
[{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
command = apachedex_every_23_hour
[apachedex-parameters]
......
......@@ -284,7 +284,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base]
{% if has_frontend -%}
......
......@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }}
# can be fully restored.
# master-data: use value "2" as we are not in a replication case
#}
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
{# KEEP GLOB PATTERN IN SYNC with generated filenames above
# YYYYmmddHHMMSS -#}
file-glob = ??????????????.sql.gz
......@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
slowquery = ${monitor-directory:private}/slowquerydex
slowquery = ${monitor-directory:private}/slowquery_digest
[{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency
......
......@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}'
if [ ! -d "$OUTPUT_FOLDER" ]; then
echo "ERROR: output_folder don't exists"
exit 0
exit 1
fi
OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
TODAY=`date +%Y%m%d`
dashed_today=$(date +%Y-%m-%d)
today=$(date -d "$dashed_today" +%Y%m%d)
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
SLOW_LOG="$SLOW_QUERY_PATH-$today"
OUTPUT_FILE="$OUTPUT_FOLDER/slowquery_digest.txt-$dashed_today"
if [ ! -f "$SLOW_LOG" ]; then
echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
exit 1
fi
$PT_QUERY_EXEC $SLOW_LOG > $OUTPUT_FILE
echo "ok"
"$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \
echo "Report generated successfully." || \
echo "Report failed with code $?"
......@@ -132,4 +132,5 @@ PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.73
pyasn1 = 0.3.7
......@@ -53,6 +53,9 @@ allow-hosts +=
www.dabeaz.com
www.owlfish.com
# Use an https index
index = https://pypi.python.org/simple/
# XXX: Workaround of SlapOS limitation
# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
# development / fast switching environment for whole software
......@@ -88,6 +91,7 @@ eggs =
slapos.libnetworkcache
[versions]
setuptools = 33.1.1
# Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos011
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
......@@ -108,7 +112,7 @@ collective.recipe.template = 2.0
cryptography = 2.1.1
decorator = 4.0.11
idna = 2.2
inotifyx = 0.2.2
inotify-simple = 1.1.1
itsdangerous = 0.24
lock-file = 2.0
lxml = 3.7.3
......@@ -122,7 +126,6 @@ pyOpenSSL = 17.2.0
pyparsing = 2.2.0
pytz = 2016.10
requests = 2.13.0
setuptools = 33.1.1
six = 1.10.0
slapos.cookbook = 1.0.53
slapos.core = 1.4.3
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment