Commit 893346ab authored by Julien Muchembled's avatar Julien Muchembled

Update Release Candidate

parents 2c79f171 b59a44d3
[buildout]
extends =
../apache/buildout.cfg
../python-2.7/buildout.cfg
../bzip2/buildout.cfg
../gdbm/buildout.cfg
../gettext/buildout.cfg
../libexpat/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
../readline/buildout.cfg
../sqlite3/buildout.cfg
../zlib/buildout.cfg
../file/buildout.cfg
parts =
apache-wsgi
[python2.7]
configure-options =
--enable-shared
--disable-static
--enable-ipv6
--enable-unicode=ucs4
--with-system-expat
--with-threads
environment =
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
LD_RUN_PATH=${buildout:parts-directory}/${:_buildout_section_name_}/lib
[apache-wsgi]
recipe = hexagonit.recipe.cmmi
url = http://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz
md5sum = f42d69190ea0c337ef259cbe8d94d985
configure-options =
--with-apxs=${apache:location}/bin/apxs
--with-python=${python2.7:location}/bin/python2.7
environment =
CPPFLAGS = -I${python2.7:location}/include/python2.7
LDFLAGS = -L${python2.7:location}/lib
......@@ -72,48 +72,49 @@ gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64
java_home = ${zulu:location}
bazelrc = ${:_profile_base_location_}/bazelrc
script =
extract_dir = self.extract(self.download(self.options['url'], self.options['md5sum']))
install =
import os, shutil
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
crosstool_path = os.path.join(extract_dir, 'tools', 'cpp', 'CROSSTOOL')
os.chmod(crosstool_path, 0644)
shutil.copy(self.options['bazel-crosstool-modified-file-path'],
shutil.copy(options['bazel-crosstool-modified-file-path'],
crosstool_path)
src_main_cpp_build_path = os.path.join(extract_dir, 'src', 'main', 'cpp', 'BUILD')
os.chmod(src_main_cpp_build_path, 0644)
shutil.copy(self.options['bazel-src-main-cpp-build-path'],
shutil.copy(options['bazel-src-main-cpp-build-path'],
src_main_cpp_build_path)
src_main_tools_build_path = os.path.join(extract_dir, 'src', 'main', 'tools', 'BUILD')
os.chmod(src_main_tools_build_path, 0644)
shutil.copy(self.options['bazel-src-main-tools-build-path'],
shutil.copy(options['bazel-src-main-tools-build-path'],
src_main_tools_build_path)
src_tools_singlejar_build_path = os.path.join(extract_dir, 'src', 'tools', 'singlejar', 'BUILD')
os.chmod(src_tools_singlejar_build_path, 0644)
shutil.copy(self.options['bazel-src-tools-singlejar-build-path'],
shutil.copy(options['bazel-src-tools-singlejar-build-path'],
src_tools_singlejar_build_path)
target_path = extract_dir+'/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java'
os.chmod(target_path, 0644)
call([self.buildout['bazel']['patch-binary'], '-p1', '-d', extract_dir, '-i', self.buildout['bazel']['patch-file-path']])
path = ':'.join((
self.options['unzip-bin'],
self.options['zip-bin'],
self.options['gcc-bin'],
options['unzip-bin'],
options['zip-bin'],
options['gcc-bin'],
os.environ['PATH']
))
env = {'JAVA_HOME':self.options['java_home'],
env = {'JAVA_HOME':options['java_home'],
'PATH':path,
'LD_LIBRARY_PATH':':'.join((
self.options['gcc-lib'],
self.options['gcc-lib64'],
options['gcc-lib'],
options['gcc-lib64'],
os.environ.get('LD_LIBRARY_PATH', '')
)),
'LDFLAGS':'-Wl,-rpath='+self.options['gcc-lib64'],
'CC':self.options['gcc-bin']+'/gcc',
'CXX':self.options['gcc-bin']+'/g++',
'BAZELRC':self.options['bazelrc'],
'LDFLAGS':'-Wl,-rpath='+options['gcc-lib64'],
'CC':options['gcc-bin']+'/gcc',
'CXX':options['gcc-bin']+'/g++',
'BAZELRC':options['bazelrc'],
'VERBOSE':'yes',
}
bin_dir = os.path.join(self.options['location'], 'bin')
bin_dir = os.path.join(location, 'bin')
os.makedirs(bin_dir)
call(['bash', 'compile.sh', 'compile'], cwd=extract_dir, env=env)
shutil.copy(os.path.join(extract_dir, 'output', 'bazel'),
......
......@@ -19,11 +19,7 @@ url = http://www.boa.org/boa-0.94.14rc21.tar.gz
md5sum = e24b570bd767a124fcfb40a34d148ba9
patches =
${boa-patch-ENOSYS:location}/${boa-patch-ENOSYS:filename}
slapos_promise =
directory:bin/
file:bin/boa
file:bin/boa_indexer
script =
install =
import shutil
import os
url = self.download(self.options['url'], self.options['md5sum'])
......@@ -33,10 +29,11 @@ script =
call(['./configure'], cwd=workdir)
call(['make'], cwd=workdir)
# Installation of boa. Manually, no make install
os.makedirs('%(location)s/bin/')
shutil.copyfile(workdir + '/src/boa', '%(location)s/bin/boa')
os.chmod('%(location)s/bin/boa', 0755)
shutil.copyfile(workdir + '/src/boa_indexer', '%(location)s/bin/boa_indexer')
os.chmod('%(location)s/bin/boa_indexer', 0755)
bindir = os.path.join(location, 'bin')
os.makedirs(bindir)
for name in 'boa', 'boa_indexer':
path = os.path.join(bindir, name)
shutil.copyfile(os.path.join(workdir, 'src', name), path)
os.chmod(path, 0755)
environment =
PATH=${patch:location}/bin:%(PATH)s
......@@ -9,13 +9,14 @@ extends =
[boost-lib]
recipe = slapos.recipe.cmmi
shared = true
url = http://downloads.sourceforge.net/sourceforge/boost/boost_1_58_0.tar.bz2
md5sum = b8839650e61e9c1c0a89f371dd475546
url = http://downloads.sourceforge.net/sourceforge/boost/boost_1_67_0.tar.bz2
md5sum = ced776cb19428ab8488774e1415535ab
location = @@LOCATION@@
configure-command = ./bootstrap.sh --prefix=${:location} --with-python=${python2.7:location}/bin/python2.7
make-binary =
make-options =
make-targets = ./b2 link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib install
make-targets = for a in $MAKEFLAGS; do case $a in -j*) j=$a; break;; esac; done;
./b2 $j link=shared dll-path=${:location}/lib:${bzip2:location}/lib:${gcc:location}/lib:${gcc:location}/lib64:${zlib:location}/lib install
environment =
PATH=${gcc:location}/bin:%(PATH)s
BZIP2_INCLUDE=${bzip2:location}/include
......
......@@ -10,16 +10,17 @@ url = http://git.busybox.net/busybox/snapshot/busybox-1_20_2.tar.gz
md5sum = 025acebb48040ef62dd635d416d317e8
patches =
${:_profile_base_location_}/busybox-1_20_2.patch#292498db86c46e101bb14bf2c74c36f0
script =
extract_dir = self.extract(self.download(%(url)r, %(md5sum)r))
install =
env = self.environ
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
workdir = guessworkdir(extract_dir)
self.applyPatchList(self.options.get('patches'), '-p1', cwd=workdir)
self.applyPatchList(options.get('patches'), '-p1', cwd=workdir)
self.logger.info("Creating default configuration")
call(['make', 'defconfig'], cwd=workdir, env=env)
self.logger.info("Building")
call(['make'], cwd=workdir, env=env)
self.logger.info("Installing")
call(['make', 'CONFIG_PREFIX=%(location)s', 'install'], cwd=workdir, env=env)
call(['make', 'CONFIG_PREFIX=' + location, 'install'], cwd=workdir, env=env)
self.logger.info("Installation finished")
environment =
PATH=${patch:location}/bin:%(PATH)s
......@@ -26,10 +26,11 @@ part = ${chromedriver:location}
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
script =
chromedriver = self.options['part']
install =
import os
chromedriver = options['part']
part = self.buildout[os.path.basename(chromedriver)]
with open(%(location)r, 'w') as f:
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
......@@ -65,10 +66,9 @@ library =
${nss:location}/lib
${nspr:location}/lib
path =
script =
url, md5sum = self.options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
os.mkdir(%(location)r)
shutil.copy(extract_dir + '/chromedriver', %(location)r)
install =
import os, shutil
extract_dir = self.extract(self.download(*options[guessPlatform()].split()))
os.mkdir(location)
shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
......@@ -46,10 +46,11 @@ part = ${chromium:location}
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
script =
chromeium = self.options['part']
part = self.buildout[os.path.basename(chromeium)]
with open(%(location)r, 'w') as f:
install =
import os
chromium = options['part']
part = self.buildout[os.path.basename(chromium)]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
# this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
......@@ -59,7 +60,7 @@ script =
export PATH={}:$PATH
exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
""".format(
chromeium,
chromium,
':'.join(part['library'].split()),
':'.join(part['path'].split()),
))
......@@ -103,9 +104,6 @@ md5sum-x86_64 =
recipe = slapos.recipe.build
slapos_promise =
file:chrome
x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
......@@ -153,13 +151,12 @@ library =
${zlib:location}/lib
path =
${fontconfig:location}/bin
script =
install =
url, md5sum = self.options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), %(location)r)
self.copyTree(guessworkdir(extract_dir), location)
# XXX adjust some permissions
import os
os.system('"${findutils:location}/bin/find" "%(location)s" -type d -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
os.system('"${findutils:location}/bin/find" "%(location)s" -type f -executable -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
os.system('"${findutils:location}/bin/find" "%(location)s" -type f -exec "${coreutils:location}/bin/chmod" a+r {} \;')
os.system("'${findutils:location}/bin/find' '%s' -type d -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -executable -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -exec '${coreutils:location}/bin/chmod' a+r {} \\;" % location)
......@@ -38,9 +38,8 @@ depends =
x86 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/28b36a94ad1a405bac689a76b6c353a7/condor-7.9.5-x86_Debian6-unstripped.tar.gz 227059bb9bebc9033665bb246d90a7bb
x86-64 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/7000a12f9a22765b457e2958762871d5/condor-7.9.5-x86_64_Debian6-stripped.tar.gz 30fa4cebabb8ff4971c769f4ef74ba68
script =
if not self.options.get('path'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
install =
url, md5sum = self.options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
\ No newline at end of file
self.copyTree(workdir, location)
......@@ -15,15 +15,10 @@ url_x86 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_386.zip
md5sum_x86 = dfdc0eedd79baab7e6bc56c1582fd02e
md5sum_x86-64 = d6bc0898ea37ae2198370a9e1978d1bb
# script to install.
script =
location = %(location)r
self.failIfPathExists(location)
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
install =
import shutil
platform = guessPlatform()
url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform]
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
shutil.move(extract_dir, location)
......@@ -20,7 +20,7 @@ make-options =
post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#d7985eef1ceef01af00ffb1d38708c8d:post_make_hook
environment =
PATH=${patch:location}/bin:%(PATH)s
randomsleep_install = ${randomsleep:location}
dummy = ${randomsleep:recipe}
[dcron-output]
# Shared binary location to ease migration
......
......@@ -31,10 +31,11 @@ part = ${firefox:location}
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
script =
firefox = self.options['part']
install =
import os
firefox = options['part']
part = self.buildout[os.path.basename(firefox)]
with open(%(location)r, 'w') as f:
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
......@@ -157,10 +158,10 @@ library =
path =
${fontconfig:location}/bin
script =
url, md5sum = self.options[guessPlatform()].split()
install =
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), %(location)r)
self.copyTree(guessworkdir(extract_dir), location)
[geckodriver]
......@@ -218,7 +219,8 @@ location = ${buildout:bin-directory}/${:_buildout_section_name_}
x86 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
x86-64 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
script =
url, md5sum = self.options[guessPlatform()].split()
install =
import shutil
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
shutil.copy(extract_dir + '/geckodriver', %(location)r)
shutil.copy(extract_dir + '/geckodriver', location)
[buildout]
extends =
../boost-lib/buildout.cfg
../tokyocabinet/buildout.cfg
parts =
flare
[flare]
recipe = slapos.recipe.cmmi
url = http://labs.gree.jp/data/source/flare-1.0.9.tgz
md5sum = e59ccce1ba29e7edf6f665561678d5c8
configure-options =
--with-tokyocabinet=${tokyocabinet:location}
--with-boost=${boost-lib:location}
environment =
LDFLAGS =-Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${boost-lib:location}/lib
......@@ -63,19 +63,25 @@ md5sum = d0efec10b9f110a32e9b8f796e21782c
[msttcore-fonts]
location = ${fonts:location}/${:_buildout_section_name_}
recipe = slapos.recipe.build
script =
install =
import os, subprocess
from zc.buildout.download import Download
d = self.options['location']
d = location
fonts = []
download = lambda x, dl=Download(self.buildout['buildout']): (
dl("http://downloads.sf.net/corefonts/%%s32.exe" %% name, md5sum=md5sum)
dl("http://downloads.sf.net/corefonts/%s32.exe" % name, md5sum=md5sum)
for md5sum, name in (x.split() for x in x.splitlines() if x))
extract = lambda x, d=d, p7z="${p7zip:location}/bin/7z": any(
subprocess.check_call((p7z, "x", "-ssc-", path, "*.ttf"), cwd=d)
for path, is_temp in x)
try: fonts += download(self.options['fonts']); os.makedirs(d); extract(fonts)
except: shutil.rmtree(d, ignore_errors=True); raise
finally: any(os.remove(path) for path, is_temp in fonts if is_temp)
try:
fonts += download(options['fonts'])
os.makedirs(d)
extract(fonts)
finally:
for path, is_temp in fonts:
if is_temp:
os.remove(path)
slapos_promise =
slapos_update_promise = ${:slapos_promise}
fonts =
......
......@@ -25,8 +25,8 @@ parts =
[cairo]
recipe = slapos.recipe.cmmi
url = http://cairographics.org/releases/cairo-1.14.2.tar.xz
md5sum = e1cdfaf1c6c995c4d4c54e07215b0118
url = https://cairographics.org/releases/cairo-1.16.0.tar.xz
md5sum = f19e0353828269c22bd72e271243a552
pkg_config_depends = ${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${glib:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libpng:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
configure-options =
--disable-static
......@@ -46,8 +46,8 @@ environment =
[harfbuzz]
recipe = slapos.recipe.cmmi
url = http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-2.3.1.tar.bz2
md5sum = 531de9df7c8a5405dd9c6a873fcee8c2
url = https://www.freedesktop.org/software/harfbuzz/release/harfbuzz-2.6.4.tar.xz
md5sum = 2b3a4dfdb3e5e50055f941978944da9f
pkg_config_depends = ${cairo:location}/lib/pkgconfig:${cairo:pkg_config_depends}:${icu4c:location}/lib/pkgconfig
configure-options =
--disable-static
......@@ -63,15 +63,15 @@ environment =
[fribidi]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/fribidi/fribidi/releases/download/v1.0.5/fribidi-1.0.5.tar.bz2
md5sum = 0f97f65038545340316ec0f4bd53df0b
url = https://github.com/fribidi/fribidi/releases/download/v1.0.9/fribidi-1.0.9.tar.xz
md5sum = 1b767c259c3cd8e0c8496970f63c22dc
configure-options =
--enable-shared
--disable-static
[pango]
recipe = slapos.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/core/3.32/3.32.2/sources/pango-1.42.4.tar.xz
url = https://download.gnome.org/sources/pango/1.42/pango-1.42.4.tar.xz
md5sum = deb171a31a3ad76342d5195a1b5bbc7c
pkg_config_depends = ${fribidi:location}/lib/pkgconfig:${harfbuzz:location}/lib/pkgconfig:${harfbuzz:pkg_config_depends}
configure-options =
......@@ -86,8 +86,8 @@ environment =
[gdk-pixbuf]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/gdk-pixbuf-2.36.0.tar.xz
md5sum = 1a3baf91956c7923dab49ee3de100ce1
url = https://download.gnome.org/sources/gdk-pixbuf/2.36/gdk-pixbuf-2.36.7.tar.xz
md5sum = 04ca1535f3fab7070db48009ed1e93f6
pkg_config_depends = ${glib:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libX11:pkg_config_depends}:${pcre:location}/lib/pkgconfig
configure-options =
--disable-static
......@@ -104,8 +104,8 @@ environment =
[atk]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/atk-2.22.0.tar.xz
md5sum = c7f2adcf75e4058727174cde970e9129
url = https://download.gnome.org/sources/atk/2.28/atk-2.28.1.tar.xz
md5sum = dfb5e7474220afa3f4ca7e45af9f3a11
configure-options =
--with-python=${python2.7:location}/bin/python2.7
--disable-gtk-doc-html
......@@ -117,8 +117,8 @@ environment =
[gtk-2]
recipe = slapos.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/gtk+-2.24.31.tar.xz
md5sum = 526a1008586094a2cbb4592fd3f9ee10
url = https://download.gnome.org/sources/gtk+/2.24/gtk+-2.24.32.tar.xz
md5sum = d5742aa42275203a499b59b4c382a784
pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig
configure-options =
--disable-static
......
......@@ -27,8 +27,8 @@ extends =
[gsl]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gsl/gsl-2.3.tar.gz
md5sum = 905fcbbb97bc552d1037e34d200931a0
url = ftp://ftp.gnu.org/gnu/gsl/gsl-2.6.tar.gz
md5sum = bda73a3dd5ff2f30b5956764399db6e7
configure-options =
--disable-static
environment =
......@@ -39,8 +39,8 @@ environment =
[inkscape]
recipe = slapos.recipe.cmmi
url = https://inkscape.org/gallery/item/10682/inkscape-0.92.1.tar_XlpI7qT.bz2
md5sum = db2eb2a566cf35ff949fc9ccb172889a
url = https://inkscape.org/gallery/item/13330/inkscape-0.92.4_A6N0YOn.tar.bz2
md5sum = ac30f6d5747fd9c620c00dad500f414f
location = @@LOCATION@@
pkg_config_depends = ${freetype:location}/lib/pkgconfig:${gtkmm:location}/lib/pkgconfig:${gtkmm:pkg_config_depends}:${gsl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig
configure-command = ${cmake:location}/bin/cmake
......
......@@ -21,11 +21,11 @@ slapos_promisee =
# http://java.com/en/download/manual_java7.jsp
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97798 90a6b9e2a32d06c18a3f16b485f0d1ea
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97800 7605134662f6c87131eca5745895fe84
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
install =
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
self.copyTree(workdir, location)
[java-re-8]
recipe = slapos.recipe.build
......@@ -40,11 +40,11 @@ slapos_promisee =
# Update 161
x86 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230530_2f38c3b165be4555a1fa6e98c45e0808 32db95dd417fd7949922206b2a61aa19
x86-64 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230532_2f38c3b165be4555a1fa6e98c45e0808 4385bc121b085862be623f4a31e7e0b4
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
install =
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
self.copyTree(workdir, location)
[java-re-8-output]
# Shared binary location to ease migration
......
......@@ -8,13 +8,12 @@ parts = jsl
recipe = slapos.recipe.build
url = http://www.javascriptlint.com/download/jsl-0.3.0-src.tar.gz
md5sum = 2b94ffa4fab07acabe0c5e73cd49bcdf
script =
location = %(location)r
self.failIfPathExists(location)
install =
import os
import sys
url = self.options['url']
md5sum = self.options['md5sum']
url = options['url']
md5sum = options['md5sum']
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
os.chdir(os.path.join(workdir, 'src'))
os.system('make -f Makefile.ref -j 1 DIST="%(location)s" all export')
os.system("make -f Makefile.ref -j 1 DIST='%s' all export" % location)
......@@ -70,7 +70,6 @@ scripts =
[jupyter-notebook-initialized-scripts]
recipe = zc.recipe.egg:scripts
eggs = ${jupyter:eggs}
interpreter = pythonjupyter
environment = jupyter-env
scripts =
......
......@@ -18,6 +18,6 @@ url = ${kerberos-package:location}/krb5-1.11.1.tar.gz
configure-command = src/configure
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
make-options = -j1
environment =
PATH=${bison:location}/bin:%(PATH)s
\ No newline at end of file
......@@ -7,7 +7,6 @@ parts = libcap
[libcap2]
recipe = slapos.recipe.build
format = no
url = http://pkgs.fedoraproject.org/lookaside/pkgs/libcap/libcap-2.22.tar.bz2/ce64058bdb3f086ddbfca8ce6c919845/libcap-2.22.tar.bz2
md5sum = ce64058bdb3f086ddbfca8ce6c919845
attr-include = ${attr:location}/include/
......@@ -19,17 +18,17 @@ slapos_promise =
file:lib/libcap.so
file:sbin/getcap
file:sbin/setcap
script =
install =
import os
url = self.download(self.options['url'], self.options['md5sum'])
url = self.download(options['url'], options['md5sum'])
extract_dir = self.extract(url)
workdir = guessworkdir(extract_dir)
cflags = '-I%(attr)s' % {'attr': self.options['attr-include']}
ldflags = '-L%(attr)s -Wl,-rpath=%(attr)s' % {'attr': self.options['attr-lib']}
call(['make', 'CFLAGS=%s' % cflags, 'LDFLAGS=%s' % ldflags, 'DESTDIR=%s' % self.options['location'], 'RAISE_SETFCAP=no', 'prefix=', 'install'],
cwd=workdir, env=env)
lib64 = os.path.join(self.options['location'], 'lib64')
lib = os.path.join(self.options['location'], 'lib')
cflags = '-I%(attr-include)s' % options
ldflags = '-L%(attr-lib)s -Wl,-rpath=%(attr-lib)s' % options
call(['make', 'CFLAGS=' + cflags, 'LDFLAGS=' + ldflags, 'DESTDIR=' + location, 'RAISE_SETFCAP=no', 'prefix=', 'install'],
cwd=workdir, env=self.environ)
lib64 = os.path.join(location, 'lib64')
lib = os.path.join(location, 'lib')
# XXX: Dirty if case
# if lib64 exists, then create a symlink from lib to lib64
os.path.exists(lib64) and os.symlink(lib64, lib)
......
......@@ -5,7 +5,7 @@ parts =
[libjpeg]
recipe = slapos.recipe.cmmi
shared = true
url = http://www.ijg.org/files/jpegsrc.v8d.tar.gz
md5sum = 52654eb3b2e60c35731ea8fc87f1bd29
url = http://www.ijg.org/files/jpegsrc.v9d.tar.gz
md5sum = 693a4e10906e66467ca21f045547fe15
configure-options =
--disable-static
......@@ -17,28 +17,31 @@ md5sum_x86-64 = cbea6cd17063b5bcbe0cb32f7819f0cf
# where office code can be found?
officedir = libreoffice5.2
# script to install
script =
location = %(location)r
self.failIfPathExists(location)
install =
import os
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
platform = guessPlatform()
url = self.options['url'] %% (ARCH_DIR_MAP[platform], platform)
md5sum = self.options['md5sum_' + platform]
url = options['url'] % (ARCH_DIR_MAP[platform], platform)
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
storagedir = os.path.join(workdir, 'storage')
os.mkdir(storagedir)
rpmsdir = os.path.join(workdir, [q for q in os.listdir(workdir) if q == 'RPMS'][0])
rpmlist = [os.path.join(rpmsdir, q) for q in os.listdir(rpmsdir) if q.endswith('.rpm') and 'javafilter' not in q and 'xsltfilter' not in q]
[self.pipeCommand([[sys.executable, '${:rpm2cpio}', rpm], ['${:cpio}', '-idum']], cwd=storagedir) for rpm in rpmlist]
self.copyTree(os.path.join(storagedir, 'opt', '${:officedir}'), location, ['ure-link'])
rpmsdir = os.path.join(workdir, 'RPMS')
for q in os.listdir(rpmsdir):
if q.endswith('.rpm') and 'javafilter' not in q and 'xsltfilter' not in q:
self.pipeCommand((
(sys.executable, options['rpm2cpio'], os.path.join(rpmsdir, q)),
(options['cpio'], '-idum'),
), cwd=storagedir)
self.copyTree(os.path.join(storagedir, 'opt', options['officedir']),
location, ['ure-link'])
os.symlink('ure', os.path.join(location, 'ure-link'))
# backward compatibility for cloudooo configuration
os.mkdir(os.path.join(location, 'basis-link'))
os.symlink(os.path.join('..', 'program'), os.path.join(location, 'basis-link', 'program'))
os.symlink(os.path.join('..', 'program'),
os.path.join(location, 'basis-link', 'program'))
# helper binaries
cpio = ${cpio:location}/bin/cpio
......
......@@ -4,7 +4,6 @@ extends =
../libtool/buildout.cfg
../pkgconfig/buildout.cfg
../python-2.7/buildout.cfg
../python-setuptools/buildout.cfg
parts =
nghttp2
......@@ -12,10 +11,8 @@ parts =
[nghttp2]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/nghttp2/nghttp2/archive/v1.20.0.zip
md5sum = e77319851b27c2366aa1cc8a8eabe73e
location = @@LOCATION@@
url = https://github.com/nghttp2/nghttp2/archive/v1.40.0.tar.gz
md5sum = ef3c6983e0de2f16b523460c812fe598
pre-configure =
autoreconf -fisv -I ${libtool:location}/share/aclocal -I ${pkgconfig:location}/share/aclocal
automake
......
......@@ -7,22 +7,18 @@ parts =
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
url_x86-64 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_amd64.zip
url_x86 = https://dl.bintray.com/mitchellh/packer/packer_0.7.5_linux_386.zip
url_x86-64 = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_linux_amd64.zip
url_x86 = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_linux_386.zip
version = 0.7.5
# supported architectures md5sums
md5sum_x86 = a545108a0ccfde7c1e74de6c4e6fdded
md5sum_x86-64 = f343d709b84db494e8d6ec38259aa4a6
# script to install.
script =
location = %(location)r
self.failIfPathExists(location)
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
WK_SUFIX_MAP = { 'x86': '386', 'x86-64': 'amd64' }
install =
import shutil
platform = guessPlatform()
url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform]
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
shutil.move(extract_dir, location)
......@@ -17,20 +17,17 @@ depends =
x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686.tar.bz2 9c1426eef5b04679d65198b1bdd6ef88
x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
script =
if not self.options.get('url'):
self.options['url'], self.options['md5sum'] = \
self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'],
self.options.get('md5sum')))
install =
import os
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
with open(wrapper_location, 'w') as wrapper:
wrapper.write("""#!/bin/sh
self.copyTree(workdir, location)
with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
wrapper.write("""#!/bin/sh -e
cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/phantomjs "$@"
""")
os.chmod(wrapper_location, 0o755)
""" % options)
os.fchmod(wrapper.fileno(), 0o755)
......@@ -69,8 +69,8 @@ filename = get-pip.py
mode = 0755
[pyodide-script]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
build_dir = ${buildout:parts-directory}/pyodide-script-test
part_dir = ${buildout:parts-directory}
git-executable = ${git:location}/bin/git
......@@ -87,28 +87,26 @@ gcc_bin_dir = ${gcc-8.2:location}/bin
python_bin_dir = ${python3.7:location}/bin
md5sum = 47ec6a091e503349d948760dc03bed51
pip_script = ${get-pip:target}
script =
install =
import subprocess, os, shutil
env = {
'PKG_CONFIG_PATH':self.options['PKG_CONFIG_PATH'],
'PATH':self.options['PATH'] + ':' + os.environ['PATH'],
'CPATH':self.options['CPATH'],
'CPPFLAGS':self.options['CPPFLAGS'],
'LDFLAGS':self.options['LDFLAGS'],
'PKG_CONFIG_PATH':options['PKG_CONFIG_PATH'],
'PATH':options['PATH'] + ':' + os.environ['PATH'],
'CPATH':options['CPATH'],
'CPPFLAGS':options['CPPFLAGS'],
'LDFLAGS':options['LDFLAGS'],
}
if not os.path.isdir(self.options['build_dir']):
command_list_0 = [{'cmd': self.options['git-executable'] + ' clone ' + self.options['repository'] + ' pyodide-script-test', 'cwd': self.options['part_dir']}]
# put the Popen in the loop to let the pipe.wait() work.
if not os.path.isdir(options['build_dir']):
command_list_0 = [{'cmd': options['git-executable'] + ' clone ' + options['repository'] + ' pyodide-script-test', 'cwd': options['part_dir']}]
for command_0 in command_list_0:
pipe = subprocess.Popen(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
pipe.wait()
subprocess.check_call(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
# else:
# shutil.rmtree(self.options['build_dir'], ignore_errors=True)
# shutil.rmtree(options['build_dir'], ignore_errors=True)
cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + self.options['openssl_location'] + ' --prefix@" ' + self.options['build_dir'] + '/cpython/Makefile'
pip_script = self.options['pip_script']
cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + options['openssl_location'] + ' --prefix@" ' + options['build_dir'] + '/cpython/Makefile'
pip_script = options['pip_script']
install_pip = "python3 " + pip_script
install_pytest = "pip install pytest selenium pytest-instafail pytest-timeout"
# Hack: create a gfortran symlink inside the python bin dir.
......@@ -119,28 +117,27 @@ script =
# So we are not going to provide the custom gcc bin path in the PATH environment variable.
# But scipy needs gfortran. The workaround is just give a symlink in the cpython bin dir, which points to the custom gcc gfortran.
# Note: the parts/cpython is for run the pyodide building script. The pyodide/cpython is for compile to webassembly module which will loads to the browser.
create_gfortran_symlink = 'ln -s ' + self.options['gcc_bin_dir'] + '/gfortran ' + self.options['python_bin_dir'] + '/gfortran'
create_gfortran_symlink = 'ln -s ' + options['gcc_bin_dir'] + '/gfortran ' + options['python_bin_dir'] + '/gfortran'
command_list = [
{'cmd': cpython_makefile_command, 'cwd': self.options['build_dir']},
{'cmd': install_pip, 'cwd': self.options['build_dir']},
{'cmd': install_pytest, 'cwd': self.options['build_dir']},
{'cmd': create_gfortran_symlink, 'cwd': self.options['build_dir']},
{'cmd': 'make -C emsdk', 'cwd': self.options['build_dir']},
{'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': self.options['build_dir']},
{'cmd': 'make -C lz4', 'cwd': self.options['build_dir']},
{'cmd': 'make -C cpython', 'cwd': self.options['build_dir']},
{'cmd': 'make -C CLAPACK', 'cwd': self.options['build_dir']},
{'cmd': 'make -C packages', 'cwd': self.options['build_dir']},
{'cmd': cpython_makefile_command, 'cwd': options['build_dir']},
{'cmd': install_pip, 'cwd': options['build_dir']},
{'cmd': install_pytest, 'cwd': options['build_dir']},
{'cmd': create_gfortran_symlink, 'cwd': options['build_dir']},
{'cmd': 'make -C emsdk', 'cwd': options['build_dir']},
{'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': options['build_dir']},
{'cmd': 'make -C lz4', 'cwd': options['build_dir']},
{'cmd': 'make -C cpython', 'cwd': options['build_dir']},
{'cmd': 'make -C CLAPACK', 'cwd': options['build_dir']},
{'cmd': 'make -C packages', 'cwd': options['build_dir']},
]
for command in command_list:
pipe = subprocess.Popen(command['cmd'], cwd=command['cwd'], env=env, shell=True)
pipe.wait()
subprocess.check_call(command['cmd'], cwd=command['cwd'], env=env, shell=True)
# We need to execute the command "./emsdk/emsdk/emsdk_env.sh" to add the emsdk related environment variables.
# Run "emsdk_env.sh" first, then read the content of "emsdk_set_env.sh" to get the environment variables
emsdk_env_content = open(self.options['emsdk_set_env_script'])
emsdk_env_content = open(options['emsdk_set_env_script'])
for content in emsdk_env_content:
# remove "export " prefix
content = content[7:]
......@@ -153,13 +150,12 @@ script =
env['CPATH'] += env['EMSCRIPTEN'] + '/system/include/'
command_list_2 = [
{'cmd': 'make build/pyodide.asm.js', 'cwd': self.options['build_dir']},
{'cmd': 'make -e', 'cwd': self.options['build_dir']},
{'cmd': 'make build/pyodide.asm.js', 'cwd': options['build_dir']},
{'cmd': 'make -e', 'cwd': options['build_dir']},
]
for command_2 in command_list_2:
pipe = subprocess.Popen(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
pipe.wait()
subprocess.check_call(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
[python3.7-PyYAML]
recipe = slapos.recipe.cmmi
......
......@@ -32,15 +32,15 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[python-cocoapi-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${python-cocoapi-repository:location}/PythonAPI
python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
gcc-location = ${gcc:location}
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
python_bin = self.options['python-bin']
gcc_location = self.options['gcc-location']
workdir = options['workdir']
python_bin = options['python-bin']
gcc_location = options['gcc-location']
env = {'PATH':':'.join([gcc_location+'/bin',
os.environ['PATH']]),
'CC':gcc_location+'/bin/gcc',
......@@ -50,16 +50,17 @@ script =
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[python-cocoapi-build-install-egg]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
python-cocoapi-repository-path = ${python-cocoapi-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-python-cocoapi-build = ${python-cocoapi-build:location}
egg = pycocotools
script =
install =
import os
os.makedirs(location)
workdir = self.options['python-cocoapi-repository-path']+'/PythonAPI'
workdir = options['python-cocoapi-repository-path']+'/PythonAPI'
egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[buildout]
parts = python-setuptools
[setuptools-download]
recipe = hexagonit.recipe.download
ignore-existing = true
download-only = true
package_suffix = setuptools-18.4
filename = ${:package_suffix}.tar.gz
url = https://pypi.python.org/packages/source/s/setuptools/${:filename}
md5sum = 214c6c43bd7035e870c1beab402c48e7
mode = 0644
[python-setuptools]
# Hack to manually install setuptools outside of the Buildout environment.
# It is useful in case of using things like "python setup.py test" in eggs
# downloaded from source (so not using bildout magic).
# To be able to use it, you should in your instance do an:
# export PYTHONPATH=${:location} (defined in ${:environment})
location = ${buildout:parts-directory}/${:_buildout_section_name_}
pythonpath = ${:location}/${setuptools-download:package_suffix}
environment = export PATH=${python2.7:location}/bin:$PATH; export PYTHONPATH=${:pythonpath};
stop-on-error = true
recipe = plone.recipe.command
update = true
# chmod is a hack, but the mode of hexagonit.recipe.download above didn't worked yet (there was new version recently)
command = ${:environment} (mkdir ${:location}; cd ${:location} && tar -xzvf ${setuptools-download:location}/${setuptools-download:filename} && cd ${:pythonpath} && ${buildout:executable} setup.py build) || (rm -fr ${:location}; exit 1)
......@@ -44,7 +44,6 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[pytorch-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${pytorch-repository:location}
python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
no-cuda = 0
......@@ -53,16 +52,17 @@ git-bin = ${git:location}/bin
binutils-location = ${binutils:location}
gcc-location = ${gcc:location}
openblas-location = ${openblas:location}
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
python_bin = self.options['python-bin']
binutils_location = self.options['binutils-location']
gcc_location = self.options['gcc-location']
openblas_location = self.options['openblas-location']
workdir = options['workdir']
python_bin = options['python-bin']
binutils_location = options['binutils-location']
gcc_location = options['gcc-location']
openblas_location = options['openblas-location']
env = {'PYTHONPATH':workdir,
'PATH':':'.join([self.options['cmake-bin'],
self.options['git-bin'],
'PATH':':'.join([options['cmake-bin'],
options['git-bin'],
binutils_location+'/bin',
gcc_location+'/bin',
os.environ['PATH']]),
......@@ -84,26 +84,27 @@ script =
'-Wl,-rpath,'+openblas_location+'/lib',
])
}
if self.options.get('no-cuda') == '1':
if options.get('no-cuda') == '1':
env['NO_CUDA'] = '1'
import os.path
call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[pytorch-build-install-egg]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
pytorch-repository-path = ${pytorch-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-pytorch-build = ${pytorch-build:location}
egg = torch
script =
install =
import os
os.makedirs(location)
workdir = self.options['pytorch-repository-path']
workdir = options['pytorch-repository-path']
egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[pytorch-egg]
recipe = zc.recipe.egg
......
......@@ -33,13 +33,9 @@ library =
${libffi:location}/lib
${zlib:location}/lib
${libuuid:location}/lib
script =
import glob
location = %(location)r
self.failIfPathExists(location)
url = self.options['url']
md5sum = self.options['md5sum']
extract_dir = self.extract(self.download(url, md5sum))
install =
import glob, os, shutil
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
os.mkdir(location)
source_file = glob.glob(os.path.join(extract_dir, '*', 'client-linux-debug'))[0]
shutil.copy(source_file, location)
......@@ -49,7 +45,7 @@ script =
export LD_LIBRARY_PATH={}
exec {}/client-linux-debug "$@"
""".format(
':'.join(self.options['library'].split()),
':'.join(options['library'].split()),
location
))
os.fchmod(f.fileno(), 0o755)
......@@ -4,26 +4,12 @@ extends =
../bash/buildout.cfg
[randomsleep]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin_dir = ${buildout:bin-directory}
bash_script_code =
if [ "$#" -ne 1 ]; then
echo "usage: randomsleep maxseconds"
exit
fi
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
template =
inline:#!${bash:location}/bin/bash
[ $# = 1 ] || {
echo "usage: ${:_buildout_section_name_} maxseconds"
exit 1
}
exec ${coreutils:location}/bin/sleep $((RANDOM * $1 >> 15)).$((RANDOM * 100 >> 15))
wrapper_script_code =
#!${bash:location}/bin/bash
. ${:location}/randomsleep.bash
script =
os.makedirs(self.options['location'])
bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
with open(bash_script_path, 'w') as f:
f.write(self.options['bash_script_code'])
wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
with open(wrapper_script_path, 'w') as f:
f.write(self.options['wrapper_script_code'])
os.chmod(wrapper_script_path, 0o750)
......@@ -4,20 +4,16 @@ parts = spidermonkey
[spidermonkey-1.7]
recipe = slapos.recipe.build
url = ftp://ftp.mozilla.org/pub/mozilla.org/js/js-1.7.0.tar.gz
url = http://ftp.mozilla.org/pub/mozilla.org/js/js-1.7.0.tar.gz
md5sum = 5571134c3863686b623ebe4e6b1f6fe6
slapos_promise =
directory:include/
directory:bin/
file:bin/js
statlib:lib/libjs.a
file:lib/libjs.so
script =
url = self.download(self.options['url'], self.options['md5sum'])
install =
import os
url = self.download(options['url'], options['md5sum'])
extract_dir = self.extract(url)
workdir = os.path.join(guessworkdir(extract_dir), 'src')
env.update(JS_DIST=self.options['location'])
call(['make', '-f', 'Makefile.ref'], cwd=workdir, env=env)
env = self.environ
env['JS_DIST'] = location
call(['make', '-f', 'Makefile.ref', '-j1'], cwd=workdir, env=env)
call(['make', '-f', 'Makefile.ref', 'export'], cwd=workdir, env=env)
[spidermonkey]
......
[buildout]
extends =
buildout.hash.cfg
../python3/buildout.cfg
parts =
surykatka
[surykatka-requirements]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[surykatka]
recipe = plone.recipe.command
command =
bash -c "${python3.7:executable} -m venv ${:location} && \
. ${:location}/bin/activate && \
pip install -r ${surykatka-requirements:target}"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
executable = ${:location}/bin/surykatka
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[surykatka-requirements]
_update_hash_filename_ = requirements.txt
md5sum = c4bd35bcc5c9c25efe1edff5dd022605
Click==7.0
certifi==2019.11.28
chardet==3.0.4
dnspython==1.16.0
forcediphttpsadapter==1.0.1
idna==2.8
miniupnpc==2.0.2
peewee==3.13.1
requests==2.22.0
surykatka==0.4.2
urllib3==1.25.8
......@@ -19,7 +19,6 @@ command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:
[tensorboard-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${tensorboard-repository:location}
gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib
......@@ -28,22 +27,22 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin
bazel-bin = ${bazel:location}/bin
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
env = {'PATH':':'.join([self.options['gcc-bin'],
self.options['java_home_bin'],
self.options['bazel-bin'],
workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'],
options['bazel-bin'],
os.environ['PATH']]),
'COMPILER_PATH':':'.join([self.options['gcc-bin'],
'COMPILER_PATH':':'.join([options['gcc-bin'],
os.environ.get('COMPILER_PATH') or '']),
'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
self.options['gcc-lib64'],
'LIBRARY_PATH':':'.join([options['gcc-lib'],
options['gcc-lib64'],
os.environ.get('LIBRARY_PATH') or '']),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
'PYTHON_LIB_PATH':self.options['python27-lib'],
'PYTHON_BIN_PATH':options['numpy-python-command'],
'PYTHON_LIB_PATH':options['python27-lib'],
}
import os.path
env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
call(bazel_command, cwd=workdir, env=env)
......@@ -52,24 +51,24 @@ script =
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorboard-repository-path = ${tensorboard-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location}
egg = tensorflow-tensorboard
bazel-bin = ${bazel:location}/bin
java_home_bin = ${bazel:java_home}/bin
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script =
install =
import os
os.makedirs(location)
workdir = self.options['tensorboard-repository-path']
workdir = options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'],
self.options['java_home_bin'],
env = {'PATH':':'.join([options['bazel-bin'],
options['java_home_bin'],
os.environ['PATH']]),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
'PYTHON_BIN_PATH':options['numpy-python-command'],
}
call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions]
[buildout]
# [buildout]
extends =
../../stack/slapos.cfg
../scipy/buildout.cfg
......@@ -43,7 +43,6 @@ cudnn_install_path = /usr/local/cuda
[tensorflow-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${tensorflow-repository:location}
gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib
......@@ -53,20 +52,21 @@ python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin
bazel_bin = ${bazel:location}/bin
need_cuda = ${cuda:cuda_toolkit_path}
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
env = {'PATH':':'.join([self.options['gcc-bin'],
self.options['java_home_bin'],
self.options['bazel_bin'],
workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'],
options['bazel_bin'],
os.environ['PATH']]),
'COMPILER_PATH':':'.join([self.options['gcc-bin'],
'COMPILER_PATH':':'.join([options['gcc-bin'],
os.environ.get('COMPILER_PATH') or '']),
'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
self.options['gcc-lib64'],
'LIBRARY_PATH':':'.join([options['gcc-lib'],
options['gcc-lib64'],
os.environ.get('LIBRARY_PATH') or '']),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
'PYTHON_LIB_PATH':self.options['python27-lib'],
'PYTHON_BIN_PATH':options['numpy-python-command'],
'PYTHON_LIB_PATH':options['python27-lib'],
'CC_OPT_FLAGS':'-march=native',
'TF_NEED_JEMALLOC':'1',
'TF_ENABLE_XLA':'0',
......@@ -79,7 +79,7 @@ script =
'TF_NEED_MPI':'0',
'TF_NEED_CUDA':self.buildout['cuda']['tf_need_cuda'],
##### FOR CUDA #####
'GCC_HOST_COMPILER_PATH':os.path.join(self.options['gcc-bin'], 'gcc'),
'GCC_HOST_COMPILER_PATH':os.path.join(options['gcc-bin'], 'gcc'),
'TF_CUDA_VERSION':self.buildout['cuda']['tf_cuda_version'],
'CUDA_TOOLKIT_PATH':self.buildout['cuda']['cuda_toolkit_path'],
'TF_CUDNN_VERSION':self.buildout['cuda']['tf_cudnn_version'],
......@@ -87,10 +87,9 @@ script =
'TF_CUDA_COMPUTE_CAPABILITIES':self.buildout['cuda']['tf_cuda_compute_capabilities'],
####################
}
import os.path
if not os.path.exists(env['CUDA_TOOLKIT_PATH']): env['TF_NEED_CUDA'] = '0'
env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
env['LDFLAGS'] = '-lW,-rpath='+self.options['gcc-lib64']
env['LDFLAGS'] = '-lW,-rpath='+options['gcc-lib64']
call(['./configure'], cwd=workdir, env=env)
bazel_command = (env['TF_NEED_CUDA'] == '1' and
['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '--config', 'cuda', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package']
......@@ -99,21 +98,20 @@ script =
call(bazel_command, cwd=workdir, env=env)
[tensorflow-build-install-egg]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorflow-repository-path = ${tensorflow-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location}
need-tensorflow-build = ${tensorflow-build:location}
need-protobuf-python = ${protobuf-python:egg}
egg = tensorflow
script =
install =
import os
os.makedirs(location)
workdir = self.options['tensorflow-repository-path']
workdir = options['tensorflow-repository-path']
egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions]
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
......@@ -7,27 +7,22 @@ parts =
tomcat
[tomcat]
<= tomcat6
[tomcat6]
recipe = hexagonit.recipe.download
ignore-existing = true
strip-top-level-dir = true
url = http://apache.multidist.com/tomcat/tomcat-6/v6.0.36/bin/apache-tomcat-6.0.36.tar.gz
md5sum = 3dde098fd0b3a08d3f2867e4a95591ba
<= tomcat9
[tomcat7]
recipe = hexagonit.recipe.download
ignore-existing = true
strip-top-level-dir = true
url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.100/bin/apache-tomcat-7.0.100.tar.gz
url = https://archive.apache.org/dist/tomcat/tomcat-7/v${:version}/bin/apache-tomcat-${:version}.tar.gz
version = 7.0.100
md5sum = 79be4ba5a6e770730a4be3d5cb3c7862
[tomcat9]
recipe = hexagonit.recipe.download
ignore-existing = true
strip-top-level-dir = true
url = https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.12/bin/apache-tomcat-9.0.12.tar.gz
url = https://archive.apache.org/dist/tomcat/tomcat-9/v${:version}/bin/apache-tomcat-${:version}.tar.gz
version = 9.0.12
md5sum = 7283da4a3a6e939adcd8f919be4ba41a
[tomcat7-output]
......
......@@ -23,33 +23,24 @@ url_x86 = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux
md5sum_x86 = ce1a2c0b2cf786ccc5d5828c42c99ddd
md5sum_x86-64 = 96b7306cebb9e65355f69f7ab63df68b
# script to install.
script =
location = %(location)r
self.failIfPathExists(location)
import sys
import os
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
install =
import os,shutil, sys, tempfile
platform = guessPlatform()
url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform]
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
path = self.download(url, md5sum)
import tempfile
extract_dir = tempfile.mkdtemp(self.name)
self.cleanup_dir_list.append(extract_dir)
self.logger.debug('Created working directory ' + repr(extract_dir))
self.logger.debug('Created working directory %s', extract_dir)
env = os.environ.copy()
env["PATH"] = "${tar:location}/bin:${xz-utils:location}/bin" + (":" + env["PATH"] if env.get("PATH") else "")
env["LD_LIBRARY_PATH"] = "${xz-utils:location}/lib" + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "")
call(["tar", "xJf", path, "-C", extract_dir], env=env)
shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
wrapper_location = os.path.join("%(location)s", "wkhtmltopdf")
wrapper = open(wrapper_location, 'w')
wrapper.write("""#!${dash:location}/bin/dash
with open(os.path.join(location, "wkhtmltopdf"), 'w') as wrapper:
wrapper.write("""#!/bin/sh
export LD_LIBRARY_PATH=%(location)s/lib:${fontconfig:location}/lib:${freetype:location}/lib:${libX11:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libpng12:location}/lib:${zlib:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/wkhtmltopdf "$@"
""")
wrapper.close()
os.chmod(wrapper_location, 0755)
""" % options)
os.fchmod(wrapper.fileno(), 0o755)
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.139'
version = '1.0.143'
name = 'slapos.cookbook'
long_description = open("README.rst").read()
......@@ -103,7 +103,6 @@ setup(name=name,
'dropbear = slapos.recipe.dropbear:Recipe',
'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
'dropbear.client = slapos.recipe.dropbear:Client',
'duplicity = slapos.recipe.duplicity:Recipe',
'equeue = slapos.recipe.equeue:Recipe',
'erp5.promise = slapos.recipe.erp5_promise:Recipe',
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
......@@ -129,7 +128,6 @@ setup(name=name,
'logrotate.d = slapos.recipe.logrotate:Part',
'mkdirectory = slapos.recipe.mkdirectory:Recipe',
'mioga.instantiate = slapos.recipe.mioga.instantiate:Recipe',
'mydumper = slapos.recipe.mydumper:Recipe',
'nbdserver = slapos.recipe.nbdserver:Recipe',
'neoppod.cluster = slapos.recipe.neoppod:Cluster',
'neoppod.admin = slapos.recipe.neoppod:Admin',
......
......@@ -27,7 +27,6 @@
#
##############################################################################
import errno
import io
import logging
import os
import sys
......@@ -124,21 +123,6 @@ class GenericBaseRecipe(object):
def createExecutable(self, name, content, mode=0o700):
return self.createFile(name, content, mode)
def addLineToFile(self, filepath, line, encoding='utf8'):
"""Append a single line to a text file, if the line does not exist yet.
line must be unicode."""
if os.path.exists(filepath):
lines = [l.rstrip('\n') for l in io.open(filepath, 'r', encoding=encoding)]
else:
lines = []
if not line in lines:
lines.append(line)
with io.open(filepath, 'w+', encoding=encoding) as f:
f.write(u'\n'.join(lines))
def createPythonScript(self, name, absolute_function, args=(), kw={}):
"""Create a python script using zc.buildout.easy_install.scripts
......
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
from slapos.recipe.librecipe import GenericBaseRecipe
def _mydumper_base_cmd(mydumper, database, user, password,
socket=None, host=None, port=None, **kw):
cmd = [mydumper]
cmd.extend(['-B', database])
if socket:
cmd.extend(['-S', socket])
else:
cmd.extend(['-h', host])
cmd.extend(['-P', port])
cmd.extend(['-u', user])
if password:
cmd.extend(['-p', password])
return cmd
def do_export(args):
cmd = _mydumper_base_cmd(**args)
if args['compression']:
cmd.append('--compress')
if args['rows'] is not None:
cmd.extend(['-r', args['rows']])
cmd.extend(['-o', args['directory']])
os.execv(cmd[0], cmd)
def do_import(args):
cmd = _mydumper_base_cmd(**args)
cmd.append('--overwrite-tables')
cmd.extend(['-d', args['directory']])
os.execv(cmd[0], cmd)
class Recipe(GenericBaseRecipe):
def install(self):
config = {
'database': self.options['database'],
'directory': self.options['backup-directory'],
'user': self.options['user'],
'password': self.options.get('password'),
}
if self.options.get('host'):
config['host'] = self.options['host']
config['port'] = self.options.get('port', 3306)
elif self.options.get('socket'):
config['socket'] = self.options['socket']
else:
raise ValueError("host or socket must be defined")
if self.optionIsTrue('import', False):
function = do_import
config['mydumper'] = self.options['myloader-binary']
else:
function = do_export
config['mydumper'] = self.options['mydumper-binary']
config['compression'] = self.optionIsTrue('compression', default=False)
config['rows'] = self.options.get('rows')
return self.createPythonScript(self.options['wrapper'],
'%s.%s' % (function.__module__, function.__name__), (config,))
......@@ -46,20 +46,16 @@ class Recipe(GenericBaseRecipe):
class Callback(GenericBaseRecipe):
def createCallback(self, notification_id, callback):
def install(self):
options = self.options
notification_id = options['on-notification-id']
# XXX: hashing the name here and in
# slapos.toolbox/slapos/pubsub/__init__.py is completely messed up and
# prevent any debug.
callback_id = sha512(str2bytes(notification_id)).hexdigest()
filepath = os.path.join(self.options['callbacks'], callback_id)
self.addLineToFile(filepath, callback)
return filepath
def install(self):
# XXX this path is returned multiple times, one for each callback that has been added.
return [self.createCallback(self.options['on-notification-id'],
self.options['callback'])]
return self.createFile(os.path.join(options['directory'], callback_id),
options['callbacks'])
class Notify(GenericBaseRecipe):
......
......@@ -43,7 +43,7 @@ from slapos.recipe.librecipe import shlex
def promise(ssh_client, user, host, port):
# Redirect output to /dev/null
with open(os.devnull) as _dev_null:
with open(os.devnull, 'wb') as _dev_null:
ssh = subprocess.Popen(
(ssh_client, '%s@%s' % (user, host), '-p', str(port)),
stdin=subprocess.PIPE, stdout=_dev_null, universal_newlines=True)
......
......@@ -42,7 +42,7 @@ md5sum = 432d22bb0f67df5203bbc5d1134a952b
[template-update-rss-script]
filename = template-update-rss.sh.in
md5sum = 80b3746568bc8e308a1f337bdaa2441c
md5sum = ae4a0043414336a521b524d9c95f1c68
[template-pullrdiffbackup]
filename = instance-pullrdiffbackup.cfg.in
......
......@@ -16,8 +16,6 @@ extends =
../../stack/monitor/buildout.cfg
parts =
extra-eggs
monitor-extra-eggs
rdiff-backup
dcron
logrotate
......@@ -30,17 +28,12 @@ parts =
template-crontab-line
slapos-cookbook
[monitor-extra-eggs]
[rssgen-eggs]
recipe = zc.recipe.egg
eggs =
${monitor-eggs:eggs}
[extra-eggs]
recipe = zc.recipe.egg
interpreter = pythonforrssgen
interpreter = python-${:_buildout_section_name_}
eggs =
PyRSS2Gen
python_dateutil
python-dateutil
##########################################################
# Service startup scripts and configuration files
......@@ -112,7 +105,7 @@ mode = 0644
rdiff-backup = 1.0.5+SlapOSPatched001
gunicorn = 19.1.1
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.2
slapos.recipe.template = 2.4.3
PyRSS2Gen = 1.1
gitdb2 = 2.0.5
smmap2 = 2.0.5
......
......@@ -4,7 +4,7 @@ STATUS_DIR=$${directory:status}
RSS_DIR=$${directory:www}
TAIL=${coreutils-output:tail}
PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
PYTHON=${buildout:bin-directory}/${rssgen-eggs:interpreter}
STATUS2RSS=${status2rss:output}
BASENAME=${coreutils-output:basename}
......
......@@ -118,4 +118,4 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[template-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 7814a48cc05d25c8f2d7b527ef2485b5
md5sum = 9d6111a5d6bc07e708116ca331925241
......@@ -210,7 +210,7 @@ ca-certificate = ${directory:etc-kedifa}/ca-certificate.pem
crl = ${directory:etc-kedifa}/crl.pem
template-csr = ${directory:etc-kedifa}/template-csr.pem
pidfile = ${directory:run}/kedifa.pid
logfile = ${directory:log}/kedfia.log
logfile = ${directory:log}/kedifa.log
[kedifa-reloader]
recipe = slapos.cookbook:wrapper
......
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
\ No newline at end of file
T-1/var/log/kedifa.log
\ No newline at end of file
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-cache-direct/_replicate_access_log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedfia.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
......
......@@ -18,4 +18,4 @@ md5sum = 6e4431cf4b0a0d034402604b1e2844c0
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = 79f2740f7e28ad9fcd2f823d076e790e
md5sum = 5fc0919cc3eab365f773c6eb73f9c6c3
......@@ -117,10 +117,12 @@ crl = ${apache-ssl-client:crl}
[apache-promise]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
<= monitor-promise-base
module = check_port_listening
module = check_url_available
name = apache.py
config-hostname = {{ ipv4 }}
config-port = {{ apache_dict.values()[0][0] }}
config-url = https://{{ ipv4 }}:{{ apache_dict.values()[0][0] }}
# XXX cloudooo replies "400 Bad Request" for GET on / but what we want to check
# is that we don't have a "503 Service Unavailable" from apache or haproxy.
config-http_code = 400
[apache-conf-ssl]
cert = ${directory:apache-conf}/apache.crt
......
Tests for Cloudooo software release
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
......@@ -24,26 +24,29 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
from setuptools import setup, find_packages
class Recipe(GenericBaseRecipe):
version = '0.0.1.dev0'
name = 'slapos.test.cloudooo'
with open("README.md") as f:
long_description = f.read()
def install(self):
remote_url = self.options['remote-backup']
backup_directory = self.options['local-directory']
cmd = [self.options['duplicity-binary'],]
options = ['--no-encryption', '--archive-dir', self.options['cache']]
if self.optionIsTrue('recover', False):
options.append('--force')
# duplicity [options] remote backup
cmd.extend(options)
cmd.extend([remote_url, backup_directory])
else:
# duplicity [options] local remote
cmd.extend(options)
cmd.extend([backup_directory, remote_url])
return self.createWrapper(self.options['wrapper'], cmd)
setup(name=name,
version=version,
description="Test for SlapOS' cloudooo",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
'six',
'PyPDF2',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
# coding: utf-8
#
# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import json
import six.moves.xmlrpc_client as xmlrpclib
import ssl
import base64
import io
import PyPDF2
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# Cloudooo needs a lot of time before being available.
CloudOooTestCase.instance_max_retry = 30
def normalizeFontName(font_name):
if '+' in font_name:
return font_name.split('+')[1]
if font_name.startswith('/'):
return font_name[1:]
def getReferencedFonts(pdf_file_reader):
"""Return fonts referenced in this pdf
"""
fonts = set()
def collectFonts(obj):
"""Recursively visit PDF objects and collect referenced fonts in `fonts`
"""
if hasattr(obj, 'keys'):
if '/BaseFont' in obj:
fonts.add(obj['/BaseFont'])
for k in obj.keys():
collectFonts(obj[k])
for page in pdf_file_reader.pages:
collectFonts(page.getObject()['/Resources'])
return {normalizeFontName(font) for font in fonts}
class HTMLtoPDFConversionFontTestMixin:
"""Mix-In class to test how fonts are selected during
HTML to PDF conversions.
This needs to be mixed with a test case defining:
* pdf_producer : the name of /Producer in PDF metadata
* expected_font_mapping : a mapping of resulting font name in pdf,
keyed by font-family in the input html
* _convert_html_to_pdf: a method to to convert html to pdf
"""
def _convert_html_to_pdf(self, src_html):
# type: (str) -> bytes
"""Convert the HTML source to pdf bytes.
"""
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def test(self):
actual_font_mapping_mapping = {}
for font, expected_substitution in sorted(
self.expected_font_mapping.items()):
src_html = '''
<style>
p {{ font-family: "{font}"; font-size: 20pt; }}
</style>
<p>the quick brown fox jumps over the lazy dog.</p>
<p>THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.</p>
'''.format(**locals())
pdf_data = self._convert_html_to_pdf(src_html)
pdf_reader = PyPDF2.PdfFileReader(io.BytesIO((pdf_data)))
self.assertEqual(
self.pdf_producer,
pdf_reader.getDocumentInfo()['/Producer'])
fonts_in_pdf = getReferencedFonts(pdf_reader)
if len(fonts_in_pdf) == 1:
actual_font_mapping_mapping[font] = fonts_in_pdf.pop()
else:
actual_font_mapping_mapping[font] = fonts_in_pdf
self.maxDiff = None
self.assertEqual(self.expected_font_mapping, actual_font_mapping_mapping)
class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
__partition_reference__ = 'wk'
pdf_producer = 'Qt 4.8.7'
expected_font_mapping = {
'Arial Black': 'Roboto-Medium',
'Arial': 'Roboto-Medium',
'Avant Garde': 'Roboto-Medium',
'Bookman': 'Roboto-Medium',
'Carlito': 'Roboto-Medium',
'Comic Sans MS': 'Roboto-Medium',
'Courier New': 'Roboto-Medium',
'DejaVu Sans Condensed': 'Roboto-Medium',
'DejaVu Sans ExtraLight': 'Roboto-Medium',
'DejaVu Sans Mono': 'Roboto-Medium',
'DejaVu Sans': 'Roboto-Medium',
'DejaVu Serif Condensed': 'Roboto-Medium',
'DejaVu Serif': 'Roboto-Medium',
'Garamond': 'Roboto-Medium',
'Gentium Basic': 'Roboto-Medium',
'Gentium Book Basic': 'Roboto-Medium',
'Georgia': 'Roboto-Medium',
'Helvetica': 'Roboto-Medium',
'Impact': 'Roboto-Medium',
'IPAex Gothic': 'Roboto-Medium',
'IPAex Mincho': 'Roboto-Medium',
'Liberation Mono': 'LiberationMono',
'Liberation Sans Narrow': 'Roboto-Medium',
'Liberation Sans': 'LiberationSans',
'Liberation Serif': 'LiberationSerif',
'Linux LibertineG': 'Roboto-Medium',
'OpenSymbol': 'Roboto-Medium',
'Palatino': 'Roboto-Medium',
'Roboto Black': 'Roboto-Medium',
'Roboto Condensed Light': 'Roboto-Medium',
'Roboto Condensed Regular': 'Roboto-Medium',
'Roboto Light': 'Roboto-Medium',
'Roboto Medium': 'Roboto-Medium',
'Roboto Thin': 'Roboto-Medium',
'Times New Roman': 'Roboto-Medium',
'Trebuchet MS': 'Roboto-Medium',
'Verdana': 'Roboto-Medium',
'ZZZdefault fonts when no match': 'Roboto-Medium',
}
def _convert_html_to_pdf(self, src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'pdf',
False,
False,
{
'encoding': 'utf-8'
},
).encode())
class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
__partition_reference__ = 'lo'
pdf_producer = 'LibreOffice 5.2'
expected_font_mapping = {
'Arial Black': 'LinuxLibertineG',
'Arial': 'LinuxLibertineG',
'Avant Garde': 'LinuxLibertineG',
'Bookman': 'LinuxLibertineG',
'Carlito': 'Carlito',
'Comic Sans MS': 'LinuxLibertineG',
'Courier New': 'LinuxLibertineG',
'DejaVu Sans Condensed': 'DejaVuSansCondensed',
'DejaVu Sans ExtraLight': 'LinuxLibertineG',
'DejaVu Sans Mono': 'DejaVuSansMono',
'DejaVu Sans': 'DejaVuSans',
'DejaVu Serif Condensed': 'DejaVuSerifCondensed',
'DejaVu Serif': 'DejaVuSerif',
'Garamond': 'LinuxLibertineG',
'Gentium Basic': 'GentiumBasic',
'Gentium Book Basic': 'GentiumBookBasic',
'Georgia': 'LinuxLibertineG',
'Helvetica': 'LinuxLibertineG',
'Impact': 'LinuxLibertineG',
'IPAex Gothic': 'IPAexGothic',
'IPAex Mincho': 'IPAexMincho',
'Liberation Mono': 'LiberationMono',
'Liberation Sans Narrow': 'LiberationSansNarrow',
'Liberation Sans': 'LiberationSans',
'Liberation Serif': 'LiberationSerif',
'Linux LibertineG': 'LinuxLibertineG',
'OpenSymbol': 'OpenSymbol',
'Palatino': 'LinuxLibertineG',
'Roboto Black': 'Roboto-Black',
'Roboto Condensed Light': 'RobotoCondensed-Light',
'Roboto Condensed Regular': 'LinuxLibertineG',
'Roboto Light': 'Roboto-Light',
'Roboto Medium': 'Roboto-Medium',
'Roboto Thin': 'Roboto-Thin',
'Times New Roman': 'LinuxLibertineG',
'Trebuchet MS': 'LinuxLibertineG',
'Verdana': 'LinuxLibertineG',
'ZZZdefault fonts when no match': 'LinuxLibertineG',
}
def _convert_html_to_pdf(self, src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'pdf',
).encode())
......@@ -18,4 +18,4 @@ md5sum = 8288e59eb442c662544daffbf446a033
[template-deploy-test]
filename = instance.cfg.in
md5sum = 616abb7fb4608321e11ade0a43b0ce4b
md5sum = b673c497abe5b3e7de4c7fc8b01aa3d0
......@@ -47,7 +47,7 @@ config-data-to-vm =
config-enable-http-server = true
# VM options
config-ram-size = 2048
config-ram-size = 4096
config-cpu-count = 2
# await for system to be ready
......
......@@ -29,7 +29,7 @@ rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
template = inline:
#!/bin/sh
export PATH=${python-with-eggs:location}:$PATH
exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory')}}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@"
exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory', 'playbook/roles/standalone-shared') }}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@"
mode = 0755
context =
key slapparameter_dict slap-configuration:configuration
......
......@@ -8,7 +8,6 @@ extends =
../../component/nodejs/buildout.cfg
parts =
monitor-eggs
slapos-cookbook
nodejs
template
......
......@@ -50,7 +50,7 @@ mode = 0644
# but '1.3.4nxd2-SlapOSPatched001'.
gunicorn = 19.1.1
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.2
slapos.recipe.template = 2.4.3
inotifyx = 0.2.2
gitdb2 = 2.0.3
smmap2 = 2.0.3
......
......@@ -15,7 +15,7 @@
[template]
filename = instance.cfg.in
md5sum = c9d4356b5148ed8ff8c4f3da63c137ce
md5sum = cad7d490b84a484b089e34b27a5675ff
[template-kvm]
filename = instance-kvm.cfg.jinja2
......
......@@ -87,7 +87,7 @@ extra-context =
raw novnc_location ${noVNC:location}
raw netcat_bin ${netcat:location}/bin/netcat
raw python_executable ${buildout:executable}
raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
raw python_eggs_executable ${buildout:bin-directory}/${extra-eggs:interpreter}
raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}
......
......@@ -19,7 +19,6 @@ extends =
# to avoid versioning issues
common-parts =
template
eggs
# XXX: we have to manually add this for resilience
rdiff-backup
......@@ -41,22 +40,12 @@ parts = ${:common-parts}
#XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
[eggs]
recipe = zc.recipe.egg
interpreter = python.eggs
eggs =
[extra-eggs]
eggs +=
${python-cffi:egg}
${python-cryptography:egg}
${lxml-python:egg}
websockify
slapos.cookbook
slapos.toolbox
erp5.util
cns.recipe.symlink
collective.recipe.template
plone.recipe.command
${pycurl:egg}
collective.recipe.shelloutput
[http-proxy]
# https://github.com/nodejitsu/node-http-proxy
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = dc9770bacea2c504b92ad2162e58d222
md5sum = 07873511a15df74f0be761947819ac9b
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = 9e237dbdda59e788202f0da194a57d41
md5sum = a57106ee88ff3295b9ffce84105da79b
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = f28a329e830ed737d468abcb4e89e1a2
md5sum = c535f4df6388cdc3f6c1df2a91d6dd53
[network-bench-cfg]
filename = network_bench.cfg.in
......@@ -38,8 +38,8 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
[monitor-collect-csv-dump]
filename = script/collect_csv_dump.py
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = a2de719a5a65438c8c3ee5195442beb6
md5sum = 89545501f0e5bf11608978886429da3d
......@@ -9,7 +9,7 @@
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %}
{%- if 'check-status-code' not in slave %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
{%- endif %}
......@@ -37,7 +37,7 @@
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- for slave in sorted(slave_instance_list) %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
......
......@@ -27,7 +27,7 @@ extra-context =
{%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %}
{%- set monitor_base_url_dict = {} -%}
{%- for slave in sorted(slave_instance_list) %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- do authorized_slave_list.append(slave) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
......@@ -56,7 +56,7 @@ software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
return = monitor-base-url
{% for section, edgebot_request in request_dict.iteritems() %}
{% for section, edgebot_request in request_dict.items() %}
[{{section}}]
<= replicate
name = {{ edgebot_request.get('name') }}
......@@ -66,11 +66,11 @@ state = {{ edgebot_request.get('state') }}
{%- set slave_configuration_dict = slapparameter_dict %}
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{%- for config_key, config_value in slave_configuration_dict.iteritems() %}
{%- for config_key, config_value in slave_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- if edgebot_request.get('sla') %}
{%- for parameter, value in edgebot_request.get('sla').iteritems() %}
{%- for parameter, value in edgebot_request.get('sla').items() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- else %}
......
......@@ -44,17 +44,12 @@ context = import json_module json
raw template_json_edgetest_test ${json-test-template:target}
mode = 0644
[surykatka-wrapped]
recipe = slapos.cookbook:wrapper
wrapper-path = $${buildout:bin-directory}/surykatka-wrapped
command-line = ${surykatka:location}/bin/python3.7 ${surykatka:executable}
[instance-base-edgebot]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
surykatka-binary = ${surykatka:executable}
surykatka-binary = ${buildout:bin-directory}/surykatka
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
......@@ -64,7 +59,7 @@ context = import json_module json
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary surykatka-wrapped:wrapper-path
key surykatka_binary :surykatka-binary
key template_surykatka_ini :template-surykatka-ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
......
......@@ -27,6 +27,7 @@
#
##############################################################################
from __future__ import print_function
import os
import argparse
import csv
......@@ -53,15 +54,14 @@ def parseArguments():
return parser.parse_args()
def writeFile(name, folder, date_scope, rows):
if os.path.exists(
os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))):
folder = os.path.join(folder, date_scope)
f = os.path.join(folder, "dump_%s.csv" % name)
if os.path.exists(f):
# File already exists, no reason to recreate it.
return
mkdir_p(os.path.join(folder, date_scope), 0o755)
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
csv_output = csv.writer(file_io)
csv_output.writerows(rows)
file_io.close()
mkdir_p(folder, 0o755)
with open(f, "w") as file_io:
csv.writer(file_io).writerows(rows)
def dump_table_into_csv(db, folder):
db.connect()
......@@ -96,6 +96,6 @@ if __name__ == "__main__":
if not os.path.exists(parser.collector_db):
print "Collector database not found..."
print("Collector database not found...")
dump_table_into_csv(Database(parser.collector_db), parser.output_folder)
......@@ -4,9 +4,11 @@ extends =
buildout.hash.cfg
../../component/pycurl/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/surykatka/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
../../component/python3/buildout.cfg
python = python3
parts =
slapos-cookbook
......@@ -68,6 +70,7 @@ eggs +=
${python-cryptography:egg}
slapos.toolbox
slapos.core
surykatka
scripts =
networkbench
......@@ -78,13 +81,30 @@ scripts =
monitor.genstatus
monitor.configwrite
check-computer-memory
surykatka
[monitor-eggs]
recipe = zc.recipe.egg
eggs +=
slapos.cookbook
slapos.recipe.cmmi
hexagonit.recipe.download
plone.recipe.command
scripts =
[versions]
slapos.recipe.template = 4.4
surykatka = 0.4.2
# For surykatka 0.4.2
click = 7.0
certifi = 2019.11.28
chardet = 3.0.4
dnspython = 1.16.0
forcediphttpsadapter = 1.0.1
idna = 2.8
miniupnpc = 2.0.2
peewee = 3.13.1
requests = 2.22.0
urllib3 = 1.25.8
......@@ -10,7 +10,7 @@ NAMESERVER =
{%- endfor %}
{% endif %}
URL =
{%- for slave in sorted(slave_instance_list) %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
......
......@@ -65,7 +65,7 @@ class ServicesTestCase(SlapOSInstanceTestCase):
self.assertIn(expected_process_name, process_names)
class MonitorTestMixin(object):
class MonitorTestMixin:
monitor_setup_url_key = 'monitor-setup-url'
def test_monitor_setup(self):
......@@ -152,8 +152,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
)
def updateSurykatkaDict(self):
class_list = self.surykatka_dict.keys()
for class_ in class_list:
for class_ in self.surykatka_dict:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,))
......@@ -200,7 +199,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
set(
glob.glob(
os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))),
set([q['ini-file'] for q in self.surykatka_dict.values()])
{q['ini-file'] for q in self.surykatka_dict.values()}
)
for info_dict in self.surykatka_dict.values():
self.assertEqual(
......@@ -243,14 +242,11 @@ class EdgeSlaveMixin(MonitorTestMixin):
for info_dict in self.surykatka_dict.values():
if os.path.exists(info_dict['json-file']):
os.unlink(info_dict['json-file'])
env = os.environ.copy()
env.pop('PYTHONPATH', None)
try:
subprocess.check_call(info_dict['status-json'], shell=True, env=env)
subprocess.check_call(info_dict['status-json'])
except subprocess.CalledProcessError as e:
self.fail('%s failed with code %s and message %s' % (
info_dict['status-json'], e.returncode, e.output))
self.assertTrue(os.path.exists(info_dict['json-file']))
with open(info_dict['json-file']) as fh:
status_json = json.load(fh)
self.assertIn('bot_status', status_json)
......@@ -680,8 +676,8 @@ INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.org/
https://www.erp5.com/"""}
https://www.erp5.com/
https://www.erp5.org/"""}
}
@classmethod
......
......@@ -18,11 +18,11 @@ md5sum = 80599fcc6e5d07270d7900aebfd62139
[root-common]
filename = root-common.cfg.in
md5sum = c03fbfc9df9edc1ef60be970e0627c5e
md5sum = 8e28f599247ad604ec6e32df410412a8
[instance-neo-admin]
filename = instance-neo-admin.cfg.in
md5sum = 87670ddc6b5d2007dac1b6d2ba86d168
md5sum = dabc1e50475055b3ee9184dcace5e8d2
[instance-neo-master]
filename = instance-neo-master.cfg.in
......@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
[instance-neo]
filename = instance-neo.cfg.in
md5sum = 512383220488335ac186013c2ffdc7c1
md5sum = 7642c760a2c5af3e3e81c2c54486d1a8
[template-neo-my-cnf]
filename = my.cnf.in
......
{%- if slapparameter_dict.get('admin') != 0 %}
{%- set monitor_dict = slapparameter_dict['monitor'] %}
{%- set monitor_dict = slapparameter_dict.get('monitor', {}) %}
{%- set bang_on_problem = monitor_dict.pop('bang-on-problem', None) %}
{%- set periodicity = monitor_dict.pop('periodicity', None) %}
{%- set monitor_passwd = slapparameter_dict.get('monitor-passwd') %}
[buildout]
extends =
{{ template_monitor }}
parts +=
logrotate-admin
neoctl
{%- if monitor_passwd != None %}
monitor-neo-health
extends =
{{ template_monitor }}
[directory]
plugin = ${:etc}/plugin
[monitor-neo-health]
<= monitor-promise-base
module = check_neo_health
name = ${:_buildout_section_name_}.py
config-neoctl = ${neoctl:wrapper-path}
{%- if bang_on_problem != None %}
config-bang-on-problem = {{ dumps(bang_on_problem) }}
{%- endif %}
{%- if periodicity != None %}
config-periodicity = {{ dumps(periodicity) }}
{%- endif %}
[monitor-instance-parameter]
monitor-httpd-port = {{ slapparameter_dict.get('monitor-port', 2052) }}
monitor-title = neo
password = {{ monitor_passwd }}
{%- endif %}
[neo-admin]
recipe = slapos.cookbook:neoppod.admin
......@@ -33,26 +57,6 @@ extra-options =
{%- endfor %}
{%- endfor %}
[directory]
plugin = ${:etc}/plugin
[monitor-neo-health]
<= monitor-promise-base
module = check_neo_health
name = ${:_buildout_section_name_}.py
config-neoctl = ${neoctl:wrapper-path}
{%- if bang_on_problem != None %}
config-bang-on-problem = {{ dumps(bang_on_problem) }}
{%- endif %}
{%- if periodicity != None %}
config-periodicity = {{ dumps(periodicity) }}
{%- endif %}
[monitor-instance-parameter]
monitor-httpd-port = {{ slapparameter_dict.get('admin', 2050) + 1 }}
monitor-title = neo
password = {{ slapparameter_dict['monitor-passwd'] }}
[logrotate-admin]
< = logrotate-entry-base
name = neo-admin
......
......@@ -82,15 +82,18 @@
"additionalProperties": false,
"properties": {
"admin": {
"description": "Port of admin node. 0 to disable.",
"default": 2050,
"description": "Admin port, 0 to disable. Default to 2050 if the node has a master, 0 otherwise.",
"type": "integer"
},
"master": {
"description": "Port of master node. 0 to disable.",
"description": "Master port, 0 to disable.",
"default": 2051,
"type": "integer"
},
"monitor": {
"description": "Monitor port, 0 to disable. Unless explicitly set by a node, the first one with an admin gets a monitor at port 2052.",
"type": "integer"
},
"storage-count": {
"description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
"default": 1,
......
......@@ -65,7 +65,7 @@ recipe = slapos.cookbook:publish.serialised
# TODO: make port a partition parameter
ip = {{ "[%s]" % list(ipv6_set)[0] if ipv6_set else list(ipv4_set)[0] }}
{% set admin = slapparameter_dict.get('admin', 2050) -%}
{% set master = slapparameter_dict.get('master', 2052) -%}
{% set master = slapparameter_dict.get('master', 2051) -%}
{% if master -%}
port-master = {{ master }}
master = ${:ip}:${:port-master}
......@@ -75,10 +75,12 @@ master =
{% if admin -%}
port-admin = {{ admin }}
admin = ${:ip}:${:port-admin}
{%- if 'monitor-passwd' in slapparameter_dict %}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
{% else -%}
{%- endif %}
{%- else %}
admin =
{% endif -%}
{%- endif %}
{#- Hack to deploy SSL certs via instance parameters #}
{%- for name, pem in zip(('ca.crt', 'neo.crt', 'neo.key'),
......
......@@ -88,13 +88,26 @@ software-type = {{ software_type }}
{% endfor -%}
config-autostart = {{ dumps(sum(storage_count)) }}
{%- do assert(replicas < len(node_list)) %}
{%- set admin_list = [] %}
{%- set monitor = set() %}
{%- for i, node in enumerate(node_list) %}
{%- set port = node.get('monitor') %}
{%- if port %}
{%- do monitor.add(node.get('admin') != 0 and port) %}
{%- endif %}
{%- endfor %}
{%- if False in monitor %}
{%- do monitor.remove(False) %}
{%- do assert(monitor) %}
{%- endif %}
{%- for i, node in enumerate(node_list) %}
{%- set section_id = prefix ~ i %}
{%- do section_id_list.append(section_id) %}
{%- if admin_list %}
{%- if node.get('master') == 0 and not node.get('monitor') %}
{%- do node.setdefault('admin', 0) %}
{%- endif %}
{%- if monitor or node.get('admin') == 0 %}
{%- do node.setdefault('monitor', 0) %}
{%- endif %}
[{{section_id}}]
<= {{ prefix }}request-common
......@@ -102,14 +115,21 @@ name = {{ section_id }}
return =
master
admin
{%- if node.get('admin') != 0 %}
{%- set port = node.pop('monitor', None) %}
{%- if port != 0 or node.get('admin') != 0 and not monitor %}
{%- if port != 0 %}
monitor-base-url
{%- if monitor_base_url_dict != None %}
{%- do monitor_base_url_dict.__setitem__('neo',
'${' ~ section_id ~ ':connection-monitor-base-url}') %}
{%- endif %}
{%- do admin_list.append(section_id) %}
{%- if monitor_base_url_dict != None %}
{%- do monitor_base_url_dict.__setitem__('neo',
'${' ~ section_id ~ ':connection-monitor-base-url}') %}
{%- endif %}
config-monitor-passwd = ${monitor-htpasswd:passwd}
{%- endif %}
{%- if port %}
config-monitor-port = {{ dumps(port) }}
{%- else %}
{%- do monitor.add(None) %}
{%- endif %}
config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
{%- endif %}
{%- for k, v in node.iteritems() %}
......@@ -117,7 +137,7 @@ config-{{ k }} = {{ dumps(v) }}
{%- endfor %}
{{ sla(section_id) }}
{%- endfor %}
{%- do assert(len(admin_list) == 1, admin_list) %}
{%- do assert(len(monitor) == 1, monitor) %}
[{{section(prefix ~ 'cluster')}}]
recipe = slapos.cookbook:neoppod.cluster
......
......@@ -25,7 +25,6 @@ parts =
# before any other section that would depend on it
neoppod-develop
neoppod
slapos-deps-eggs
slapos-cookbook
[neoppod-repository]
......@@ -72,18 +71,6 @@ egg = ${:_buildout_section_name_}
setup-eggs = ${python-cffi:egg}
###
[slapos-deps-eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
${pycurl:egg}
${python-PyYAML:egg}
${python-cryptography:egg}
${python-cliff:egg}
slapos.toolbox
scripts =
slapos-kill
[download-base-neo]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment