Commit a8f35d7e authored by Kirill Smelkov's avatar Kirill Smelkov

Merge branch 'master+ZODB4-wc2' into y/wc2-next

* master+ZODB4-wc2: (56 commits)
  golang: v↑ go1.17  (1.17.3 -> 1.17.6)
  golang: v↑ go1.16  (1.16.10 -> 1.16.13)
  stack/erp5: Shorten haproxy socket path
  software/erp5/test: fix flaky test_activity_processing
  component/apache: version up 2.4.52, including security fixes.
  component/trafficserver: version up 9.1.1.
  stack/slapos: version up lxml 4.6.5, including security fixes.
  ninja: simplify configure step
  consul, packer, phantomjs: enable shared
  version up: zc.buildout 2.7.1+slapos015, slapos.recipe.build 0.53, slapos.recipe.cmmi 0.19
  version up: CMake 3.22.1
  component/userhosts: use github.com/figiel/hosts for compatibility with recent libc
  software/erp5/upgrade_test: backport new fixes in old software
  software/erp5/upgrade_test: skip software checks
  component/coturn: build with slapos libnsl
  component/apache-php: build with slapos libnsl
  repman: improve parameters used for cluster
  component/theia: version up 1.21.0
  software/theia: update python packages in python-language-server-requirements
  component/theia: update plugins with current versions
  ...
parents 1f35a79f 7005a464
......@@ -23,6 +23,7 @@ extends =
../imagemagick/buildout.cfg
../icu/buildout.cfg
../openssl/buildout.cfg
../libnsl/buildout.cfg
[php-redis]
recipe = slapos.recipe.cmmi
......@@ -112,7 +113,7 @@ environment =
PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${libzip:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
CPPFLAGS=-I${libzip:location}/include
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib
TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
HOME=${apache:location}
......
......@@ -39,9 +39,9 @@ configure-options =
[apache]
recipe = slapos.recipe.cmmi
shared = true
version = 2.4.51
version = 2.4.52
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = d2793fc1c8cb8ba355cee877d1f2d46d
md5sum = a94ae42b84309d5ef6e613ae825b92fa
configure-options = --disable-static
--enable-authn-alias
--enable-bucketeer
......
......@@ -15,37 +15,29 @@ parts =
[chromedriver-wrapper-91]
<= chromedriver-wrapper
wrapper-name = chromedriver-91
part = ${chromedriver-91:location}
part = chromedriver-91
[chromedriver-wrapper-2.41]
<= chromedriver-wrapper
wrapper-name = chromedriver-2.41
part = ${chromedriver-2.41:location}
part = chromedriver-2.41
[chromedriver-wrapper]
# generate a wrapper named ${:wrapper-name} setting $LD_LIBRARY_PATH
# to run chromedriver installed from ${:part}
wrapper-name = chromedriver
part = ${chromedriver:location}
part = chromedriver
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
location = ${buildout:bin-directory}/${:part}
init =
self.buildout[options['part']]
install =
import os
chromedriver = options['part']
part = self.buildout[os.path.basename(chromedriver)]
part = self.buildout[options['part']]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
export PATH={}:$PATH
exec ./chromedriver "$@"
""".format(
chromedriver,
':'.join(part['library'].split()),
':'.join(part['path'].split()),
))
export LD_LIBRARY_PATH=%s
exec %s/chromedriver "$@"
""" % (':'.join(part['library'].split()), part['location']))
os.fchmod(f.fileno(), 0o755)
......@@ -67,18 +59,13 @@ md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
[chromedriver-download]
# Installs chromedriver ${version}.
# This chromedriver is not usable directly, it needs a wrapper.
version =
md5sum-x86_64 =
recipe = slapos.recipe.build
x86_64-linux-gnu = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip ${:md5sum-x86_64}
recipe = slapos.recipe.build:download-unpacked
url = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_${:_url}.zip
library =
${nss:location}/lib
${nspr:location}/lib
${libX11:location}/lib
path =
install =
import os, shutil
extract_dir = self.extract(self.download(*options[multiarch()].split()))
os.mkdir(location)
shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
[chromedriver-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux64
md5sum = ${:md5sum-x86_64}
......@@ -11,7 +11,6 @@
# in the system.
extends =
../alsa/buildout.cfg
../coreutils/buildout.cfg
../cups/buildout.cfg
../dbus/buildout.cfg
../findutils/buildout.cfg
......@@ -37,30 +36,30 @@ parts =
[chromium-wrapper]
# Install a chromium wrapper named ${:wrapper-name} in buildout's bin-directory,
# wrapping chromium installed in ${:part}
# Install a chromium wrapper named ${:part} in buildout's bin-directory,
# wrapping chromium installed in [${:part}]
# This [chromium-wrapper] installs the default chromium with name `chromium` and
# can also be used as a macro to install under a different name.
wrapper-name = chromium
part = ${chromium:location}
part = chromium
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
location = ${buildout:bin-directory}/${:part}
init =
self.buildout[options['part']]
install =
import os
chromium = options['part']
part = self.buildout[os.path.basename(chromium)]
part = self.buildout[options['part']]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
d=%s
# this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
# no system fonts are available, chrome will refuse to start with this error:
# FATAL:platform_font_linux.cc(83)] Check failed: InitDefaultFont(). Could not find the default font
export LD_LIBRARY_PATH=$PWD:{}
export PATH={}:$PATH
exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
""".format(
chromium,
export LD_LIBRARY_PATH=$d:%s
export PATH=%s:$PATH
exec $d/chrome --disable-setuid-sandbox --no-sandbox --disable-gpu "$@"
""" % (
part['location'],
':'.join(part['library'].split()),
':'.join(part['path'].split()),
))
......@@ -68,13 +67,11 @@ install =
[chromium-wrapper-91]
<= chromium-wrapper
wrapper-name = chromium-91
part = ${chromium-91:location}
part = chromium-91
[chromium-wrapper-69]
<= chromium-wrapper
wrapper-name = chromium-69
part = ${chromium-69:location}
part = chromium-69
[chromium]
......@@ -101,24 +98,7 @@ generation-x86_64 = 1532051976706023
[chromium-download]
# macro to download a binary build of chromium and generate a
# wrapper as chrome-slapos in the part directory
# the full version tag
version =
# needs the following URL parts:
# revision is from the "base position"
revision_x86-64 =
# generation is in the final download URL
generation-x86_64 =
# this is the md5sum of the downloaded archive
md5sum-x86_64 =
recipe = slapos.recipe.build
x86_64-linux-gnu = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
recipe = slapos.recipe.build:download-unpacked
library =
${atk:location}/lib
${at-spi2-atk:location}/lib
......@@ -166,12 +146,7 @@ library =
${zlib:location}/lib
path =
${fontconfig:location}/bin
install =
url, md5sum = self.options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), location)
# XXX adjust some permissions
import os
os.system("'${findutils:location}/bin/find' '%s' -type d -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -executable -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -exec '${coreutils:location}/bin/chmod' a+r {} \\;" % location)
[chromium-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
url = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media
md5sum = ${:md5sum-x86_64}
......@@ -9,8 +9,8 @@ parts =
[cmake]
recipe = slapos.recipe.cmmi
shared = true
url = https://cmake.org/files/v3.21/cmake-3.21.3.tar.gz
md5sum = c0feb5855604f68b09bdb3acb623619e
url = https://cmake.org/files/v3.22/cmake-3.22.1.tar.gz
md5sum = 83802be08c114bb34729300206488321
environment =
CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
......@@ -4,7 +4,14 @@ parts =
[consul]
recipe = slapos.recipe.build:download-unpacked
shared = true
version = 0.8.3
base = https://releases.hashicorp.com/consul/${:version}/consul_${:version}
i386-linux-gnu = ${:base}_linux_386.zip dfdc0eedd79baab7e6bc56c1582fd02e
x86_64-linux-gnu = ${:base}_linux_amd64.zip d6bc0898ea37ae2198370a9e1978d1bb
url = https://releases.hashicorp.com/consul/${:version}/consul_${:version}_${:_url}.zip
[consul:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux_386
md5sum = dfdc0eedd79baab7e6bc56c1582fd02e
[consul:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux_amd64
md5sum = d6bc0898ea37ae2198370a9e1978d1bb
......@@ -3,6 +3,7 @@ extends =
../sqlite3/buildout.cfg
../openssl/buildout.cfg
../libevent/buildout.cfg
../libnsl/buildout.cfg
parts = coturn
[coturn]
......@@ -13,4 +14,4 @@ environment =
PATH=${sqlite3:location}/bin:%(PATH)s
CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
CFLAGS=-I${libevent2:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib
author Sergey Poznyakoff <gray@gnu.org>
https://git.savannah.gnu.org/cgit/cpio.git/commit/?id=641d3f489cf6238bb916368d4ba0d9325a235afb
* src/global.c: Remove superfluous declaration of program_name
diff --git a/src/global.c b/src/global.c
index fb3abe9..acf92bc 100644
--- a/src/global.c
+++ b/src/global.c
@@ -184,9 +184,6 @@ unsigned int warn_option = 0;
/* Extract to standard output? */
bool to_stdout_option = false;
-/* The name this program was run with. */
-char *program_name;
-
/* A pointer to either lstat or stat, depending on whether
dereferencing of symlinks is done for input files. */
int (*xstat) ();
# GNU cpio copies files into or out of a cpio or tar archive. The archive can be another file on the disk, a magnetic tape, or a pipe.
[buildout]
extends =
../patch/buildout.cfg
parts = cpio
[cpio]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/cpio/cpio-2.12.tar.bz2
md5sum = 93eea9f07c0058c097891c73e4955456
url = https://ftp.gnu.org/gnu/cpio/cpio-2.13.tar.bz2
md5sum = f3438e672e3fa273a7dc26339dd1eed6
patch-options = -p1
patches =
${:_profile_base_location_}/963304-remove-superfluous-declaration-of-program_name#1aadb39448eedaf49d03cf45f9d78ea4
environment =
PATH=${patch:location}/bin:%(PATH)s
......@@ -3,6 +3,7 @@ extends =
../alsa/buildout.cfg
../dbus/buildout.cfg
../fontconfig/buildout.cfg
../fonts/buildout.cfg
../gtk-2/buildout.cfg
../gtk-3/buildout.cfg
../libffi/buildout.cfg
......@@ -22,59 +23,52 @@ parts =
[firefox-wrapper]
# Install a firefox wrapper names ${:wrapper-name} in buildout's bin-directory,
# wrapping firefox installed in ${:part}
# Install a firefox wrapper named ${:part} in buildout's bin-directory,
# wrapping firefox installed in [${:part}]
# This [firefox-wrapper] installs the default firefox with name `firefox` and
# can also be used as a macro to install under a different name.
wrapper-name = firefox
part = ${firefox:location}
part = firefox
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
location = ${buildout:bin-directory}/${:part}
fonts-conf = ${firefox-default-fonts-conf:rendered}
init =
self.buildout[options['part']]
install =
import os
firefox = options['part']
part = self.buildout[os.path.basename(firefox)]
part = self.buildout[options['part']]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
export PATH={}:$PATH
d=%s
export LD_LIBRARY_PATH=$d:%s
export PATH=%s:$PATH
# BBB use a default fonts.conf for compatibility, but it's software instance
# responsability to build a fonts.conf with the fonts they want.
[ -z $FONTCONFIG_FILE ] && export FONTCONFIG_FILE=${firefox-default-fonts-conf:rendered}
exec ./firefox "$@"
""".format(
firefox,
[ "$FONTCONFIG_FILE" ] || export FONTCONFIG_FILE=%s
exec $d/firefox "$@"
""" % (
part['location'],
':'.join(part['library'].split()),
':'.join(part['path'].split()),
options['fonts-conf'],
))
os.fchmod(f.fileno(), 0o755)
[firefox-wrapper-78]
<= firefox-wrapper
wrapper-name = firefox-78
part = ${firefox-78:location}
part = firefox-78
[firefox-wrapper-68]
<= firefox-wrapper
wrapper-name = firefox-68
part = ${firefox-68:location}
part = firefox-68
[firefox-wrapper-60]
<= firefox-wrapper
wrapper-name = firefox-60
part = ${firefox-60:location}
part = firefox-60
[firefox-wrapper-52]
<= firefox-wrapper
wrapper-name = firefox-52
part = ${firefox-52:location}
[firefox-wrapper-51]
<= firefox-wrapper
wrapper-name = firefox-51
part = ${firefox-51:location}
part = firefox-52
[firefox-default-fonts-conf]
recipe = slapos.recipe.template:jinja2
......@@ -124,20 +118,12 @@ version = 52.9.0esr
i686-md5sum = 9aa18888b7812670208490609d75c9bc
x86_64-md5sum = 9336d70f45070c743d08e5473b783a7a
[firefox-51]
<= firefox-download
version = 51.0.1
i686-md5sum = 9a5b67e9d759a1e4df004294a24b2b43
x86_64-md5sum = bd93f2652d1d90d59ae462439a93c85f
[firefox-download]
recipe = slapos.recipe.build
slapos_promise =
file:firefox
i386-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 ${:i686-md5sum}
x86_64-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ${:x86_64-md5sum}
url = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/${:_url}/en-US/firefox-${:version}.tar.bz2
library =
${alsa:location}/lib
......@@ -185,12 +171,19 @@ path =
${fontconfig:location}/bin
install =
url, md5sum = options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), location)
self.copyTree(guessworkdir(self.extract(self.download())), location)
${:post-install}
post-install =
[firefox-download:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux-i686
md5sum = ${:i686-md5sum}
[firefox-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux-x86_64
md5sum = ${:x86_64-md5sum}
[geckodriver]
# Current geckodriver installed as ${buildout:bin-directory}/geckodriver
<= geckodriver-0.24.0
......@@ -207,47 +200,26 @@ version = 0.22.0
i686-md5sum = 6de7544753fda56fbaa8382dcac99aaa
x86_64-md5sum = 81746200ce5841e00cabf3b8ea7db542
[geckodriver-0.21.0]
<= geckodriver-base
version = 0.21.0
i686-md5sum = 9fc1657dd1b94272d0cdb3b29ca80f79
x86_64-md5sum = 6bc36d4fd4975e296cdb3fa3c5e26a41
[geckodriver-0.19.0]
<= geckodriver-base
version = 0.19.0
i686-md5sum = 07cd383c8aef8ea5ef194a506141afd6
x86_64-md5sum = ca6935a72fd0527d15a78a17a35e56e8
[geckodriver-0.17.0]
<= geckodriver-base
version = 0.17.0
i686-md5sum = 79b1a158f96d29942a111c0905f1c807
x86_64-md5sum = be18faeea6e7db9db6990d8667e2298f
[geckodriver-0.16.1]
<= geckodriver-base
version = 0.16.1
i686-md5sum = not not on github
x86_64-md5sum = 57dfd55d4759d9878eb75b4c0123d00c
[geckodriver-0.14.0]
<= geckodriver-base
version = 0.14.0
i686-md5sum = b5836f5a944fe9f3ed1a67c7b342c6a7
x86_64-md5sum = 4a185d3179862a35104603b9274452e7
[geckodriver-base]
# Installs geckodriver ${version}
recipe = slapos.recipe.build
url = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-${:_url}.tar.gz
location = ${buildout:bin-directory}/${:_buildout_section_name_}
install =
import os, shutil
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
shutil.copy(os.path.join(extract_dir, 'geckodriver'), location)
i386-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
x86_64-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
[geckodriver-base:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux32
md5sum = ${:i686-md5sum}
install =
import shutil
url, md5sum = options[multiarch()].split()
extract_dir = self.extract(self.download(url, md5sum))
shutil.copy(extract_dir + '/geckodriver', location)
[geckodriver-base:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux64
md5sum = ${:x86_64-md5sum}
......@@ -59,8 +59,8 @@ environment-extra =
[golang1.16]
<= golang-common
url = https://golang.org/dl/go1.16.10.src.tar.gz
md5sum = 49f0a54f0bdcba297bac194d8dafe431
url = https://golang.org/dl/go1.16.13.src.tar.gz
md5sum = 1c076f952d9af57590a36fa7d36f695a
# go1.16 needs go1.4 to bootstrap
environment-extra =
......@@ -68,8 +68,8 @@ environment-extra =
[golang1.17]
<= golang-common
url = https://golang.org/dl/go1.17.3.src.tar.gz
md5sum = 3ea82e5966275f405f0db4f52511bb6e
url = https://golang.org/dl/go1.17.6.src.tar.gz
md5sum = dc57f93f323e9f8189e5ffc1f223e346
# go1.17 needs go1.4 to bootstrap
environment-extra =
......
......@@ -45,9 +45,9 @@ gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
# called "src".
name = src
# There's nothing special about version 92.0.4515.107. It just happened
# to be the current Chromium stable version at the time of writing.
version = 92.0.4515.107
# 96.0.4664.129 version is the latest stable version in December 2021.
# Note that we need a version compiling without python2
version = 96.0.4664.129
[headless-chromium]
......
[buildout]
extends =
../xz-utils/buildout.cfg
../libtirpc/buildout.cfg
[libnsl]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/thkukuk/libnsl/releases/download/v1.3.0/libnsl-1.3.0.tar.xz
md5sum = 9214f674bd0c2bcfdd6c1da0cadb061f
make-options =
CFLAGS=-I${libtirpc:location}/include/tirpc
LDFLAGS='-L${libtirpc:location}/lib -Wl,-rpath=${libtirpc:location}/lib -ltirpc'
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
......@@ -8,26 +8,15 @@ parts =
[libreoffice-bin]
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
version = 5.2.4.2
url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
# supported architectures md5sums
md5sum_x86 = 7a0b33a2d18f06143258c428c32de213
md5sum_x86-64 = cbea6cd17063b5bcbe0cb32f7819f0cf
url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/${:_url1}/LibreOffice_${:version}_${:_url2}_rpm.tar.gz
# where office code can be found?
officedir = libreoffice5.2
install =
import os
import sys
arch = {
'i386-linux-gnu': ('x86', 'x86'),
'x86_64-linux-gnu': ('x86_64', 'x86-64'),
}[multiarch()]
url = options['url'] % arch
md5sum = options['md5sum_' + arch[1]]
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
workdir = guessworkdir(self.extract(self.download()))
storagedir = os.path.join(workdir, 'storage')
os.mkdir(storagedir)
rpmsdir = os.path.join(workdir, 'RPMS')
......@@ -48,3 +37,13 @@ install =
# helper binaries
cpio = ${cpio:location}/bin/cpio
rpm2cpio = ${rpm2cpio:target}
[libreoffice-bin:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url1 = x86
_url2 = Linux_x86
md5sum = 7a0b33a2d18f06143258c428c32de213
[libreoffice-bin:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url1 = x86_64
_url2 = Linux_x86-64
md5sum = cbea6cd17063b5bcbe0cb32f7819f0cf
[buildout]
parts = libtirpc
[libtirpc]
recipe = slapos.recipe.cmmi
shared = true
url = https://downloads.sourceforge.net/libtirpc/libtirpc-1.3.2.tar.bz2
md5sum = cf4ca51f3fc401bea61c702c69171ab0
configure-options = --disable-gssapi
......@@ -61,8 +61,8 @@ mime = ${nginx:location}/conf/mime.types
[nginx-push-stream-module]
recipe = slapos.recipe.build:download-unpacked
shared = true
url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.4.0.tar.gz
md5sum = d9cba621b8739e13bdb5e02b9425f205
url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.5.5.tar.gz
md5sum = 9bb5237a93c957130aaf7033ed3c0989
[nginx-push-stream]
<= nginx-common
......
[buildout]
extends =
../defaults.cfg
parts =
ninja
......@@ -10,7 +8,7 @@ shared = true
version = 1.10.2
url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz
md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4
configure-command = ${python:executable} configure.py
configure-command = ${buildout:executable} configure.py
configure-options = --bootstrap --verbose
make-binary = true
post-install =
......
[buildout]
extends =
../patch/buildout.cfg
parts =
p7zip
[p7zip]
recipe = slapos.recipe.cmmi
shared = true
url = http://downloads.sf.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
version = 9.38.1
md5sum = 6cba8402ccab2370d3b70c5e28b3d651
url = https://downloads.sourceforge.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
version = 16.02
md5sum = a0128d661cfe7cc8c121e73519c54fbf
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
patch-base = https://sources.debian.org/data/main/p/p7zip/16.02%2Bdfsg-8/debian/patches/
patches =
${:patch-base}/12-CVE-2016-9296.patch#f0f5dc36d97d9a71a6da3adba717fbf2
${:patch-base}/13-CVE-2017-17969.patch#8399bdc913cf8cf8194ecdb5487f48d9
${:patch-base}/14-Fix-g%2B%2B-warning.patch#9db8fbfbc8d624669c095240621460bd
${:patch-base}/15-Fix-FTBFS-gcc10.patch#50ebfd79c715d8b5d649e57a1da2ce34
configure-command = rm -r DOC
make-binary =
make-options =
make-targets = make -j1 7z install
DEST_HOME=@@LOCATION@@
make-targets = 7z
post-install = make install DEST_HOME=%(location)s
......@@ -4,7 +4,14 @@ parts =
[packer]
recipe = slapos.recipe.build:download-unpacked
shared = true
version = 0.7.5
base = https://releases.hashicorp.com/packer/${:version}/packer_${:version}
i386-linux-gnu = ${:base}_linux_386.zip a545108a0ccfde7c1e74de6c4e6fdded
x86_64-linux-gnu = ${:base}_linux_amd64.zip f343d709b84db494e8d6ec38259aa4a6
url = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_${:_url}.zip
[packer:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux_386
md5sum = a545108a0ccfde7c1e74de6c4e6fdded
[packer:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux_amd64
md5sum = f343d709b84db494e8d6ec38259aa4a6
......@@ -7,23 +7,36 @@ parts =
[phantomjs]
recipe = slapos.recipe.build
shared = true
slapos_promise =
file:phantomjs-slapos
base = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux
i386-linux-gnu = ${:base}-i686.tar.bz2 9c1426eef5b04679d65198b1bdd6ef88
x86_64-linux-gnu = ${:base}-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
url = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-${:_url}.tar.bz2
install =
import os
extract_dir = self.extract(self.download(*options[multiarch()].split()))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, location)
self.copyTree(guessworkdir(self.extract(self.download())), location)
with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
wrapper.write("""#!/bin/sh -e
cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/phantomjs "$@"
""" % options)
export LD_LIBRARY_PATH=%s
export PATH=%s:$PATH
exec %s/bin/phantomjs "$@"
""" % (
':'.join(options['library'].split()),
':'.join(options['path'].split()),
location,
))
os.fchmod(wrapper.fileno(), 0o755)
library =
${freetype:location}/lib
${fontconfig:location}/lib
${libexpat:location}/lib
path =
${fontconfig:location}/bin
[phantomjs:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux-i686
md5sum = 9c1426eef5b04679d65198b1bdd6ef88
[phantomjs:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux-x86_64
md5sum = f278996c3edd0e8d8ec4893807f27d71
......@@ -8,6 +8,7 @@ extends =
../patch/buildout.cfg
../pcre/buildout.cfg
../cyrus-sasl/buildout.cfg
../libnsl/buildout.cfg
../m4/buildout.cfg
[postfix]
......@@ -20,7 +21,9 @@ patch-options = -p1
patches =
${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
configure-command = make
configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
configure-options = makefiles CCARGS=${:configure-options-CCARGS} AUXLIBS=${:configure-options-AUXLIBS}
configure-options-CCARGS = '-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl -I${libnsl:location}/include'
configure-options-AUXLIBS = '-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -L${libtirpc:location}/lib -L${libnsl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib -Wl,-rpath=${libnsl:location}/lib'
make-targets = non-interactive-package install_root=${:location}
environment =
PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s
......@@ -42,6 +42,7 @@ patches =
${:_profile_base_location_}/pytracemalloc_pep445.patch#9f3145817afa2b7fad801fde8447e396
${:_profile_base_location_}/disabled_module_list.patch#e038a8016475574c810cbaaf0e42f4ac
${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
${:_profile_base_location_}/py27-subproc-closefds-fast.patch#e495e44491694a8972da11739206f2e6
url =
http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
configure-options =
......
From 9a79fa64c6298998d04a5ae6d699222a93cd0cb8 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Fri, 24 Dec 2021 13:55:08 +0300
Subject: [PATCH] [py27] subprocess: Use /proc/self/fd/ to know opened file
descriptors
... to avoid O(MAXFD) slowdown on closefds=True.
This is semantic backport of what Python3 does.
---
Lib/subprocess.py | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 1f2da0ffbe8..300557220ac 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -869,7 +869,28 @@ def pipe_cloexec(self):
def _close_fds(self, but):
- if hasattr(os, 'closerange'):
+ # try to retrieve list of opened file descriptors to avoid being slow on large MAXFD
+ # https://www.erp5.com/project_section/vifib/forum/python2--subprocess.Popen-is-slow-when-close_fds-True-and-nofile-ulimit-is-high-PO01sREVXn
+ # https://bugs.python.org/issue8052
+ # https://github.com/python/cpython/commit/8facece99a59
+ fdv = None
+ try:
+ fdv = os.listdir('/proc/self/fd')
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ if fdv is not None:
+ for fd_ in fdv:
+ fd = int(fd_)
+ if (0 <= fd <= 2) or fd == but:
+ continue
+ try:
+ os.close(fd)
+ except Exception: # e.g. not KeyboardInterrupt nor SystemExit
+ pass
+
+ # fallback to closing all file descriptors in 3..MAXFD range
+ elif hasattr(os, 'closerange'):
os.closerange(3, but)
os.closerange(but + 1, MAXFD)
else:
--
2.30.2
......@@ -28,5 +28,6 @@ command = set -e
<= go-git-package
go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager
branch = 2.1
revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
#branch = 2.1
branch = develop
revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file
......@@ -19,4 +19,4 @@ md5sum = 8157c22134200bd862a07c6521ebf799
[yarn.lock]
_update_hash_filename_ = yarn.lock
md5sum = 067d2db611b21f77885f3adfd7f81453
md5sum = e0341b7a715cc757a671aef902e6767d
This diff is collapsed.
......@@ -41,8 +41,7 @@ for plugin_and_version in '''\
vscode/lua/latest
vscode/make/latest
vscode/markdown/latest
# https://github.com/eclipse-theia/theia/issues/7780
vscode/markdown-language-features/1.39.2
vscode/markdown-language-features/latest
vscode/merge-conflict/latest
vscode/npm/latest
ms-vscode/node-debug/latest
......@@ -71,13 +70,17 @@ for plugin_and_version in '''\
vscode/theme-solarized-dark/latest
vscode/theme-tomorrow-night-blue/latest
vscode/typescript/latest
vscode/typescript-language-features/latest
# latest (1.62.3) does not activate because it uses a proposed API not in theia:
# Activating extension 'TypeScript and JavaScript Language Features (built-in)' failed: r.languages.createLanguageStatusItem is not a function
vscode/typescript-language-features/1.54.3
vscode/vb/latest
vscode/vscode-theme-seti/latest
vscode/xml/latest
vscode/yaml/latest
EditorConfig/EditorConfig/latest
dbaeumer/vscode-eslint/latest
# latest (2.2.2) does not activate:
# Activating extension 'ESLint' failed: Class extends value undefined is not a constructor or null
dbaeumer/vscode-eslint/2.1.20
ms-vscode/references-view/latest
# golang.Go removed because it overwrites the PATH in theia shell
# golang/Go/0.16.2
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -24,8 +24,8 @@ min_version = 8
[trafficserver]
recipe = slapos.recipe.cmmi
url = http://apache.claz.org/trafficserver/trafficserver-9.1.0.tar.bz2
md5sum = 994b0aa879cbd95054048f34bf8ed954
url = https://dlcdn.apache.org/trafficserver/trafficserver-9.1.1.tar.bz2
md5sum = 666945652942cce8c3ff0ae2757cc3c6
shared = true
patch-options = -p1
configure-options =
......
[userhosts]
recipe = slapos.recipe.cmmi
shared = true
url = https://lab.nexedi.com/nexedi/userhosts/repository/${:revision}/archive.tar.gz
revision = a05fe5a3a5cb7005351ef4ec41460089f3ce4d0a
md5sum = 5a80b4d962d975f290a60cf790c3334d
url = https://github.com/figiel/hosts/archive/${:revision}.tar.gz
revision = fdb45fe219593d63453f4be55cfc2a1199c18f59
md5sum = 9ff5407132a2a67200ef1036a02f0ee2
configure-command = true
make-options = PREFIX=@@LOCATION@@
make-targets = check install
make-targets = test install
......@@ -4,7 +4,6 @@ extends =
../freetype/buildout.cfg
../fontconfig/buildout.cfg
../libpng/buildout.cfg
../tar/buildout.cfg
../xorg/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
......@@ -14,27 +13,39 @@ parts =
[wkhtmltopdf]
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
base = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic
i386-linux-gnu = ${:base}-i386.tar.xz ce1a2c0b2cf786ccc5d5828c42c99ddd
x86_64-linux-gnu = ${:base}-amd64.tar.xz 96b7306cebb9e65355f69f7ab63df68b
url = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_${:_url}.tar.xz
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
install =
import os,shutil, sys, tempfile
path = self.download(*options[multiarch()].split())
extract_dir = tempfile.mkdtemp(self.name)
self.cleanup_dir_list.append(extract_dir)
self.logger.debug('Created working directory %s', extract_dir)
env = os.environ.copy()
env["PATH"] = "${tar:location}/bin:${xz-utils:location}/bin" + (":" + env["PATH"] if env.get("PATH") else "")
env["LD_LIBRARY_PATH"] = "${xz-utils:location}/lib" + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "")
call(["tar", "xJf", path, "-C", extract_dir], env=env)
shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
import os
self.copyTree(guessworkdir(self.extract(self.download())), location)
with open(os.path.join(location, "wkhtmltopdf"), 'w') as wrapper:
wrapper.write("""#!/bin/sh
export LD_LIBRARY_PATH=%(location)s/lib:${fontconfig:location}/lib:${freetype:location}/lib:${libX11:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libpng12:location}/lib:${zlib:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/wkhtmltopdf "$@"
""" % options)
wrapper.write("""#!/bin/sh -e
d=%s
export LD_LIBRARY_PATH=$d/lib:%s
export PATH=%s:$PATH
exec $d/bin/wkhtmltopdf "$@"
""" % (
location,
':'.join(options['library'].split()),
':'.join(options['path'].split()),
))
os.fchmod(wrapper.fileno(), 0o755)
library =
${fontconfig:location}/lib
${freetype:location}/lib
${libX11:location}/lib
${libXext:location}/lib
${libXrender:location}/lib
${libpng12:location}/lib
${zlib:location}/lib
path =
${fontconfig:location}/bin
[wkhtmltopdf:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
_url = linux-generic-i386
md5sum = ce1a2c0b2cf786ccc5d5828c42c99ddd
[wkhtmltopdf:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
_url = linux-generic-amd64
md5sum = 96b7306cebb9e65355f69f7ab63df68b
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.220'
version = '1.0.226'
name = 'slapos.cookbook'
long_description = open("README.rst").read()
......
from __future__ import print_function
import os
import subprocess
import time
......@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
while True:
while True:
mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql_upgrade.communicate()[0]
if mysql_upgrade.returncode:
print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result)
print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
break
print "MySQL database upgraded with result:\n%s" % result
print("MySQL database upgraded with result:\n%s" % result)
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(mysql_script)[0]
if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result)
print('Command %r failed with:\n%s' % (mysql_list, result))
break
# import timezone database
mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
if mysql_tzinfo_to_sql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
break
mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True)
result = mysql.communicate(timezone_sql)[0]
if mysql.returncode:
print 'Command %r failed with:\n%s' % (mysql_list, result)
print('Command %r failed with:\n%s' % (mysql_list, result))
break
print 'SlapOS initialisation script succesfully applied on database.'
print('SlapOS initialisation script succesfully applied on database.')
return
sleep = min(sleep+1, 30)
print 'Sleeping for %ss and retrying' % sleep
print('Sleeping for %ss and retrying' % sleep)
sys.stdout.flush()
sys.stderr.flush()
time.sleep(sleep)
......@@ -49,17 +49,20 @@ def setUpModule():
class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case
"""
def getRootPartitionConnectionParameterDict(self):
@classmethod
def getRootPartitionConnectionParameterDict(cls):
"""Return the output paramters from the root partition"""
return json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
cls.computer_partition.getConnectionParameterDict()['_'])
def getComputerPartition(self, partition_reference):
for computer_partition in self.slap.computer.getComputerPartitionList():
@classmethod
def getComputerPartition(cls, partition_reference):
for computer_partition in cls.slap.computer.getComputerPartitionList():
if partition_reference == computer_partition.getInstanceParameter(
'instance_title'):
return computer_partition
def getComputerPartitionPath(self, partition_reference):
partition_id = self.getComputerPartition(partition_reference).getId()
return os.path.join(self.slap._instance_root, partition_id)
@classmethod
def getComputerPartitionPath(cls, partition_reference):
partition_id = cls.getComputerPartition(partition_reference).getId()
return os.path.join(cls.slap._instance_root, partition_id)
This diff is collapsed.
......@@ -39,13 +39,15 @@ from slapos.testing.testcase import installSoftwareUrlList
from slapos.testing.testcase import SlapOSNodeCommandError
from slapos.grid.utils import md5digest
old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.4/software/erp5/software.cfg'
old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.5/software/erp5/software.cfg'
new_software_release_url = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
_, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
old_software_release_url, software_id="upgrade_erp5")
old_software_release_url,
software_id="upgrade_erp5",
skip_software_check=True,
)
def setUpModule():
installSoftwareUrlList(
......
......@@ -35,6 +35,7 @@ import subprocess
import tempfile
import time
import six
import sys
from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
from six.moves.socketserver import StreamRequestHandler, TCPServer
......@@ -171,7 +172,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
<source>
@type exec
tag tag.name
command python %s
command %s %s
run_interval %ss
<parse>
keys pressure, humidity, temperature
......@@ -186,7 +187,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
<buffer>
flush_mode immediate
</buffer>
</match>''' % (script_path, FLUSH_INTERVAL, cls._ipv6_address)
</match>''' % (sys.executable, script_path, FLUSH_INTERVAL, cls._ipv6_address)
@classmethod
def sensor_script(cls, measurementList):
......@@ -280,4 +281,4 @@ class GatewayConfTestCase(WendelinTutorialTestCase):
data,
msgpack.unpackb(
self.serve(self._wendelin_port, WendelinHTTPRequestHandler)),
)
\ No newline at end of file
)
......@@ -19,7 +19,7 @@ md5sum = 087f226ba90928dcc5a722d7008c867a
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = b260fce887535fc69a259d1fd31af1b2
md5sum = d949e6100fd4e01819cc2867a65c6e65
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......
......@@ -1029,15 +1029,12 @@ config-command = ${ansible-vm-bin:rendered}
{% if bootstrap_script_url -%}
[download-bootstrap-script]
recipe = slapos.recipe.build
location = ${directory:public}/vm-bootstrap
install =
from zc.buildout.download import check_md5sum, urlretrieve, ChecksumError
url, md5sum = {{ repr(bootstrap_script_url) }}.split('#')
urlretrieve(url, location)
if not check_md5sum(location, md5sum):
os.remove(location)
raise ChecksumError
recipe = slapos.recipe.build:download
offline = false
destination = ${directory:public}/vm-bootstrap
{% set bootstrap_script_url = bootstrap_script_url.split('#') -%}
url = {{ bootstrap_script_url[0] }}
md5sum = {{ bootstrap_script_url[1] }}
{% endif -%}
[logrotate-vm-bootstrap]
......
......@@ -16,27 +16,27 @@
[template]
filename = instance.cfg
md5sum = 96a76300b2f714b6c47157920fe79a53
md5sum = 6ea4fa210a91c15278c847a809de5991
[template-lte-enb-epc]
_update_hash_filename_ = instance-enb-epc.jinja2.cfg
md5sum = 762d55291e75e8b61e35f9f28d29915a
md5sum = 25fae79db2b30be0f365ad967b278c3c
[template-lte-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = 83c37b43d7b70584bf71aed9289ea13c
md5sum = fe249168a3f50b0efe6aeae39afb03ae
[template-lte-gnb-epc]
_update_hash_filename_ = instance-gnb-epc.jinja2.cfg
md5sum = e43a726dd3023a4bbaa474bb2d7a6ebe
md5sum = 5ec4508b02f6e7fbdbe1f37c65b9d897
[template-lte-gnb]
_update_hash_filename_ = instance-gnb.jinja2.cfg
md5sum = fc59d15a7f7f942951f9e38d7a1cca2c
md5sum = 6deb38b3de44f6e0a43b49fd13f0f072
[template-lte-epc]
_update_hash_filename_ = instance-epc.jinja2.cfg
md5sum = 0585b4f8fd42538595e6abb61d07fd93
md5sum = 089f62e736cdc620eafc2c47b050fe13
[ue_db.jinja2.cfg]
filename = config/ue_db.jinja2.cfg
......@@ -44,11 +44,11 @@ md5sum = d33163012d6c98efc59161974c649557
[enb.jinja2.cfg]
filename = config/enb.jinja2.cfg
md5sum = d841debc51d9f12555a47d1556a6a3c1
md5sum = 97d9cdd07704cae36c5e4234c87025e8
[gnb.jinja2.cfg]
filename = config/gnb.jinja2.cfg
md5sum = da64ea9c5003f40987a8bba3f18e8839
md5sum = b0df7f3b679b25d5296dd67e074364b4
[ltelogs.jinja2.sh]
filename = ltelogs.jinja2.sh
......@@ -61,3 +61,11 @@ md5sum = 518c71ce57204304b703b977c665a164
[ims.jinja2.cfg]
filename = config/ims.jinja2.cfg
md5sum = e561ec26a70943c61557def1781cf65f
[sdr-busy-promise]
_update_hash_filename_ = promise/check_sdr_busy.py
md5sum = 9e867282d7dd80c4255e87a82a47a19d
[interface-up-promise]
_update_hash_filename_ = promise/check_interface_up.py
md5sum = 505efcbe04e717088924f2267b10c2b9
......@@ -56,15 +56,15 @@
{
/* address of MME for S1AP connection. Must be modified if the MME
runs on a different host. */
mme_addr: "127.0.1.100"
mme_addr: "{{ slapparameter_dict.get('mme_addr', '127.0.1.100') }}"
},
],
/* GTP bind address (=address of the ethernet interface connected to
the MME). Must be modified if the MME runs on a different host. */
gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
gtp_addr: "{{ gtp_addr }}",
/* high 20 bits of SIB1.cellIdentifier */
enb_id: 0x1A2D0,
enb_id: {{ slapparameter_dict.get('enb_id', '0x1A2D0') }},
/* list of cells */
cell_list: [
......
......@@ -37,15 +37,15 @@
amf_list: [
{
/* address of AMF for NGAP connection. Must be modified if the AMF runs on a different host. */
amf_addr: "127.0.1.100",
amf_addr: "{{ slapparameter_dict.get('amf_addr', '127.0.1.100') }}",
},
],
/* GTP bind address (=address of the ethernet interface connected to
the AMF). Must be modified if the AMF runs on a different host. */
gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
gtp_addr: "{{ gtp_addr }}",
gnb_id_bits: 28,
gnb_id: 0x12345,
gnb_id: {{ slapparameter_dict.get('gnb_id', '0x12345') }},
nr_support: true,
......
......@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
......@@ -34,7 +34,7 @@ promise = ${:etc}/promise
log = ${:var}/log
[request-common-base]
recipe = slapos.cookbook:request.serialised
recipe = slapos.cookbook:request
software-url = {{ slap_connection['software-release-url'] }}
server-url = {{ slap_connection['server-url'] }}
computer-id = {{ slap_connection['computer-id'] }}
......@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
key-file = {{ slap_connection['key-file'] }}
cert-file = {{ slap_connection['cert-file'] }}
sla-computer_guid = {{ slap_connection['computer-id'] }}
config-monitor-password = ${monitor-htpasswd:passwd}
return = monitor-base-url
......
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": {{ default_lte_n_rb_dl }}
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
......@@ -4,7 +4,7 @@ parts =
ltelogs
lte-enb-config
lte-enb-service
# Temporarily extend monitor-base until promises are added
sdr-busy-promise
monitor-base
publish-connection-information
......@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.network_name = RAPIDSPACE
configuration.domain = rapid.space
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
configuration.ims_ws_port = 9003
......@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
context =
section directory directory
[lte-enb-sh-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
sudo /opt/amarisoft/rm-tmp-lte | true;
{{ enb }}/lteenb ${directory:etc}/enb.cfg >> ${directory:log}/enb-output.cfg 2>> ${directory:log}/enb-output.cfg
### eNodeB (enb)
[lte-enb-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
command-line = {{ enb }}/lteenb ${directory:etc}/enb.cfg
command-line = ${lte-enb-sh-wrapper:rendered}
wrapper-path = ${directory:service}/lte-enb
mode = 0775
reserve-cpu = True
pidfile = ${directory:run}/enb.pid
hash-files =
${lte-enb-config:rendered}
${lte-enb-sh-wrapper:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
......@@ -83,6 +93,7 @@ context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
raw gtp_addr {{ local_ipv4 }}
import netaddr netaddr
[lte-enb-config]
......@@ -91,9 +102,21 @@ template = {{ enb_template }}
rendered = ${directory:etc}/enb.cfg
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
<= monitor-publish
recipe = slapos.cookbook:publish
ipv4 = {{ local_ipv4 }}
[monitor-instance-parameter]
{% if slapparameter_dict.get("name", None) %}
monitor-title = {{ slapparameter_dict['name'] | string }}
{% endif %}
{% if slapparameter_dict.get("monitor-password", None) %}
password = {{ slapparameter_dict['monitor-password'] | string }}
{% endif %}
# Add custom promise to check if /dev/sdr0 is busy
[sdr-busy-promise]
recipe = slapos.cookbook:promise.plugin
eggs = slapos.core
file = {{ sdr_busy_promise }}
output = ${directory:plugins}/check-sdr-busy.py
......@@ -4,7 +4,7 @@ parts =
ltelogs
lte-mme-config
lte-mme-service
# Temporarily extend monitor-base until promises are added
tun-up-promise
monitor-base
publish-connection-information
......@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.network_name = RAPIDSPACE
configuration.domain = rapid.space
configuration.log_size = 50M
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
......@@ -62,7 +62,7 @@ context =
[lte-ims-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/ims.log; sleep 1
command-line = rm -f ${directory:var}/lte_ue.db; {{ mme }}/lteims ${directory:etc}/ims.cfg
command-line = {{ mme }}/lteims ${directory:etc}/ims.cfg
wrapper-path = ${directory:service}/lte-ims
mode = 0775
pidfile = ${directory:run}/ims.pid
......@@ -71,17 +71,29 @@ hash-files =
{{ ue_db_path }}
environment = AMARISOFT_PATH=/opt/amarisoft/.amarisoft
[lte-mme-sh-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
rm -f ${directory:var}/lte_ue.db;
{{ mme }}/ltemme ${directory:etc}/mme.cfg >> ${directory:log}/mme-output.cfg 2>> ${directory:log}/mme-output.cfg
### MME
[lte-mme-service]
recipe = slapos.cookbook:wrapper
# When the machine shutdowns abruptly, lte_ue is not cleaned up which causes
# amarisoft ltemme to fail. TODO: find a cleaner way to handle this
init = ${ltelogs:rendered} ${directory:log}/mme.log
command-line = {{ mme }}/ltemme ${directory:etc}/mme.cfg
command-line = ${lte-mme-sh-wrapper:rendered}
wrapper-path = ${directory:service}/lte-mme
mode = 0775
pidfile = ${directory:run}/mme.pid
hash-files =
${lte-mme-config:rendered}
{{ ue_db_path }}
${lte-mme-sh-wrapper:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib:{{ nghttp2_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
......@@ -119,17 +131,18 @@ context =
import netaddr netaddr
key ifup_empty lte-mme-ifup-empty:wrapper-path
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
{% if slapparameter_dict.get("monitor-password", None) %}
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
{% else %}
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
{% endif %}
{% if slapparameter_dict.get("monitor-password", None) %}
[monitor-instance-parameter]
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif %}
[publish-connection-information]
<= monitor-publish
recipe = slapos.cookbook:publish
# Add custom promise to check if /dev/sdr0 is busy
[tun-up-promise]
recipe = slapos.cookbook:promise.plugin
eggs = slapos.core
file = {{ interface_up_promise }}
output = ${directory:plugins}/check-tun-up.py
config-ifname = ${slap-configuration:tun-name}
......@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
......@@ -34,7 +34,7 @@ promise = ${:etc}/promise
log = ${:var}/log
[request-common-base]
recipe = slapos.cookbook:request.serialised
recipe = slapos.cookbook:request
software-url = {{ slap_connection['software-release-url'] }}
server-url = {{ slap_connection['server-url'] }}
computer-id = {{ slap_connection['computer-id'] }}
......@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
key-file = {{ slap_connection['key-file'] }}
cert-file = {{ slap_connection['cert-file'] }}
sla-computer_guid = {{ slap_connection['computer-id'] }}
config-monitor-password = ${monitor-htpasswd:passwd}
return = monitor-base-url
......
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": {{ default_nr_bandwidth }}
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
......@@ -4,7 +4,7 @@ parts =
ltelogs
lte-gnb-config
lte-enb-service
# Temporarily extend monitor-base until promises are added
sdr-busy-promise
monitor-base
publish-connection-information
......@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
configuration.network_name = VIFIB
configuration.domain = vifib.com
configuration.network_name = RAPIDSPACE
configuration.domain = rapid.space
configuration.mme_ws_port = 9000
configuration.enb_ws_port = 9002
configuration.ims_ws_port = 9003
......@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
context =
section directory directory
[lte-enb-sh-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
sudo /opt/amarisoft/rm-tmp-lte | true;
{{ enb }}/lteenb ${directory:etc}/gnb.cfg >> ${directory:log}/gnb-output.cfg 2>> ${directory:log}/gnb-output.cfg
### eNodeB (enb)
[lte-enb-service]
recipe = slapos.cookbook:wrapper
init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
command-line = {{ enb }}/lteenb ${directory:etc}/gnb.cfg
command-line = ${lte-enb-sh-wrapper:rendered}
wrapper-path = ${directory:service}/lte-enb
mode = 0775
reserve-cpu = True
pidfile = ${directory:run}/enb.pid
hash-files =
${lte-gnb-config:rendered}
${lte-enb-sh-wrapper:rendered}
environment =
LD_LIBRARY_PATH={{ openssl_location }}/lib
AMARISOFT_PATH=/opt/amarisoft/.amarisoft
......@@ -83,6 +93,7 @@ context =
section directory directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
raw gtp_addr {{ local_ipv4 }}
import netaddr netaddr
[lte-gnb-config]
......@@ -91,9 +102,21 @@ template = {{ gnb_template }}
rendered = ${directory:etc}/gnb.cfg
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
<= monitor-publish
recipe = slapos.cookbook:publish
ipv4 = {{ local_ipv4 }}
[monitor-instance-parameter]
{% if slapparameter_dict.get("name", None) %}
monitor-title = {{ slapparameter_dict['name'] | string }}
{% endif %}
{% if slapparameter_dict.get("monitor-password", None) %}
password = {{ slapparameter_dict['monitor-password'] | string }}
{% endif %}
# Add custom promise to check if /dev/sdr0 is busy
[sdr-busy-promise]
recipe = slapos.cookbook:promise.plugin
eggs = slapos.core
file = {{ sdr_busy_promise }}
output = ${directory:plugins}/check-sdr-busy.py
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -26,6 +26,18 @@
"description": "number of DL resource blocks",
"type": "number",
"default": 100
},
"mme_addr": {
"title": "MME Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"enb_id": {
"title": "eNB ID",
"description": "eNB ID",
"type": "string",
"default": "0x1A2D0"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -32,6 +32,18 @@
"description": "Downlink Bandwidth (in MHz)",
"type": "number",
"default": 40
},
"amf_addr": {
"title": "AMF Address",
"description": "IPv4 of the core network",
"type": "string",
"default": "127.0.0.100"
},
"gnb_id": {
"title": "gNB ID",
"description": "gNB ID",
"type": "string",
"default": "0x12345"
}
}
}
\ No newline at end of file
......@@ -7,7 +7,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
......@@ -36,6 +36,20 @@ gnb = dynamic-template-lte-gnb:rendered
epc = dynamic-template-lte-epc:rendered
RootSoftwareInstance = $${:enb-epc}
[local-ipv4-address]
recipe = slapos.recipe.build
init =
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
s.connect(('1.1.1.1', 1))
ip = s.getsockname()[0]
except Exception:
ip = '127.0.0.1'
finally:
s.close()
options['local-ipv4'] = ip
[dynamic-template-lte-enb-epc]
< = jinja2-template-base
template = ${template-lte-enb-epc:target}
......@@ -43,6 +57,7 @@ filename = instance-lte-enb-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-gnb-epc]
< = jinja2-template-base
......@@ -51,6 +66,7 @@ filename = instance-lte-gnb-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-enb]
< = jinja2-template-base
......@@ -62,6 +78,7 @@ extra-context =
raw enb ${enb:destination}
raw enb_template ${enb.jinja2.cfg:target}
raw ltelogs_template ${ltelogs.jinja2.sh:target}
raw sdr_busy_promise ${sdr-busy-promise:target}
raw openssl_location ${openssl:location}
raw default_dl_earfcn ${enb:default-dl-earfcn}
raw default_lte_dl_freq ${enb:default-lte-dl-freq}
......@@ -71,6 +88,7 @@ extra-context =
raw default_lte_rx_gain ${enb:default-lte-rx-gain}
raw min_frequency ${enb:min-frequency}
raw max_frequency ${enb:max-frequency}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-gnb]
< = jinja2-template-base
......@@ -82,6 +100,7 @@ extra-context =
raw enb ${enb:destination}
raw gnb_template ${gnb.jinja2.cfg:target}
raw ltelogs_template ${ltelogs.jinja2.sh:target}
raw sdr_busy_promise ${sdr-busy-promise:target}
raw openssl_location ${openssl:location}
raw default_dl_nr_arfcn ${enb:default-dl-nr-arfcn}
raw default_nr_band ${enb:default-nr-band}
......@@ -91,6 +110,7 @@ extra-context =
raw default_nr_rx_gain ${enb:default-nr-rx-gain}
raw min_frequency ${enb:min-frequency}
raw max_frequency ${enb:max-frequency}
key local_ipv4 local-ipv4-address:local-ipv4
[dynamic-template-lte-epc]
< = jinja2-template-base
......@@ -99,6 +119,7 @@ filename = instance-lte-epc.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:rendered}
raw interface_up_promise ${interface-up-promise:target}
raw mme ${mme:destination}
raw mme_template ${mme.jinja2.cfg:target}
raw ims_template ${ims.jinja2.cfg:target}
......
import socket
import errno
from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
def __init__(self, config):
"""
Called when initialising the promise before testing.
Sets the configuration and the periodicity.
"""
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=2)
def sense(self):
"""
Called every time the promise is tested.
Signals a positive or negative result.
In this case, check whether the file exists.
"""
ifname = self.getConfig('ifname')
f = open('/sys/class/net/%s/operstate' % ifname, 'r')
if f.read() == 'up\n':
self.logger.info("%s is up", ifname)
else:
self.logger.error("%s is down", ifname)
f.close()
def test(self):
"""
Called after sense() if the instance is still converging.
Returns success or failure based on sense results.
In this case, fail if the previous sensor result is negative.
"""
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
"""
Called after sense() if the instance has finished converging.
Returns success or failure based on sense results.
Failure signals the instance has diverged.
In this case, fail if two out of the last three results are negative.
"""
return self._anomaly(result_count=3, failure_amount=2)
import os
import errno
from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
def __init__(self, config):
"""
Called when initialising the promise before testing.
Sets the configuration and the periodicity.
"""
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=2)
def sense(self):
"""
Called every time the promise is tested.
Signals a positive or negative result.
In this case, check whether the file exists.
"""
sdr_dev = '/dev/sdr0'
try:
open(sdr_dev, 'w').close()
self.logger.error("eNB is not using %s", sdr_dev)
except IOError as e:
if e.errno == errno.EBUSY:
self.logger.info("eNB is using %s", sdr_dev)
def test(self):
"""
Called after sense() if the instance is still converging.
Returns success or failure based on sense results.
In this case, fail if the previous sensor result is negative.
"""
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
"""
Called after sense() if the instance has finished converging.
Returns success or failure based on sense results.
Failure signals the instance has diverged.
In this case, fail if two out of the last three results are negative.
"""
return self._anomaly(result_count=3, failure_amount=2)
......@@ -49,6 +49,12 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-lte-epc]
<= download-base
[sdr-busy-promise]
<= download-base
[interface-up-promise]
<= download-base
[amarisoft]
recipe = slapos.recipe.build
path = /opt/amarisoft/lte
......
......@@ -14,19 +14,19 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = 8a08be95a04f1a47098c4fdef80bdfed
md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 697a1b546c883da45c14dbcd2d73b2b9
md5sum = 7782adde357c9444fed51f2dcc230a52
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = 2ee2c4bc5f985c11c2167b819d261256
md5sum = 0a4ce036ea524373a85fac10e81edb96
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = c78c6be9537d3dffbb526b0e819bacb5
md5sum = 8cdeb63a2acc294801266f24599f6d32
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
......@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = a5c204cac552754520aee0570d379723
md5sum = 938f1e8087395757f5ae861d20500658
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
......@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = df44c2d6fb1971df582345daedad280c
md5sum = 47b76144e1b116580c8acf08274af976
[template-publish-slave-information]
_update_hash_filename_ = publish-database-slave-parameters.cfg.in
md5sum = 9616477ff993c55c2b43bf8797db6994
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = d8a0a7d72b02f911dafda655941b805d
md5sum = 6e5c02542568f250aacd84694249efc0
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef
md5sum = c203f40a58386310a433b58fd345a341
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
......@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = d2ebd2ec55bf8489789a52c808729925
md5sum = d5c19d866c5a76d3bc2f434a91b1553a
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = e9bcee5dc1318fe3acda2663472214f5
md5sum = 8f14054b3761b167cf8a86f4f084a574
[proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in
md5sum = b45313ae5fb06972cc7fc945e34e434a
......@@ -17,17 +17,11 @@
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
database-list = {{ dumps(database_list) }}
csr-id = ${caucase-csr-id:csr-id}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -37,6 +31,22 @@ mode = 644
< = jinja2-template-base
mode = 755
[jinja2-template-script-base]
< = jinja2-template-executable
context =
key tmp_dir directory:tmp
key partition_dir buildout:directory
raw username {{ slapparameter_dict['repman-user'] }}
raw password {{ slapparameter_dict['repman-passwd'] }}
raw cluster_name {{ slapparameter_dict['cluster'] }}
raw secure_url {{ slapparameter_dict['repman-secure-url'] }}
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw jq_bin {{ jq_bin }}
raw curl_bin {{ curl_bin }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
${:extra-context}
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
......@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
{% import "caucase" as caucase with context %}
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=buildout_bin_directory,
updater_path='${directory:services}/caucase-updater',
url=slapparameter_dict['caucase-url'],
data_dir='${mysql-directory:ssl}',
crt_path='${mysql-directory:ssl}/server-cert.pem',
ca_path='${mysql-directory:ssl}/ca-cert.pem',
crl_path='${mysql-directory:ssl}/crl.pem',
key_path='${mysql-directory:ssl}/server-key.pem',
on_renew=None,
max_sleep=None,
template_csr_pem=None,
openssl=openssl_bin,
)}}
{% do part_list.append('caucase-updater') -%}
{% do part_list.append('caucase-updater-promise') -%}
[get-csr-id]
recipe = plone.recipe.command
output = ${directory:tmp}/csr_id
command =
if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
RESULT="None";
else
if [ -f "${caucase-updater-csr:csr}" ]; then
RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
if [ ! $? -eq 0 ]; then
RESULT="None";
fi
fi
fi
cat <<EOF > ${:output}
[caucase]
csr-id = $(echo $RESULT)
EOF
update-command = ${:command}
[caucase-csr-id]
recipe = slapos.cookbook:zero-knowledge.read
file-path = ${get-csr-id:output}
csr-id =
[my-cnf-parameters]
socket = ${directory:run}/mariadb.sock
ip = {{ ip }}
data-directory = ${directory:srv}/mariadb
data-directory = ${mysql-directory:data}
pid-file = ${directory:run}/mariadb.pid
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
......@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
ssl-crt = ${directory:mariadb-ssl}/crt.pem
ssl-key = ${directory:mariadb-ssl}/key.pem
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
[my-cnf]
< = jinja2-template-base
......@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
root-user = repman
heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
[init-script]
< = jinja2-template-executable
extensions = jinja2.ext.do
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
......@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('update-mysql-script') }}]
< = jinja2-template-executable
< = jinja2-template-script-base
rendered = ${directory:scripts}/mariadb_update
init-password = ${directory:etc}/.init-passwd.done
upgrade-done = ${directory:lib}/mariadb-update-done
context =
extra-context =
key init_password_done :init-password
key upgrade_done :upgrade-done
key init_root_sql init-root-sql:rendered
key mysql_update update-mysql:output
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
key init_database_sql init-script:rendered
template = {{ parameter_dict['template-init-root-wrapper'] }}
[{{ section('mysql-slave-db-cleanup') }}]
< = jinja2-template-script-base
rendered = ${directory:scripts}/manage-slave-db
db-name = {{ slapparameter_dict['database-name'] }}
extra-context =
key database_name :db-name
template = {{ parameter_dict['template-manage-db'] }}
[mysqld]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld
......@@ -136,19 +195,19 @@ environ =
{{ variable }}
{%- endfor %}
[ca-mysqld]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = ${my-cnf-parameters:ssl-key}
cert-file = ${my-cnf-parameters:ssl-crt}
executable = ${mysqld:rendered}
wrapper = ${directory:controller}/mariadb
[mysqld-launcher]
recipe = slapos.cookbook:wrapper
command-line = ${mysqld:rendered}
wrapper-path = ${directory:controller}/mariadb
wait-for-files =
${mysql-directory:mysql}/my.cnf
${mysql-directory:ssl}/server-cert.pem
{% import "supervisord_lib" as supervisord_lib with context %}
{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-mariadb-ctl") -%}
{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
"stopwaitsecs": 86400, "environment": [],
{% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
"stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
"stdout_logfile": "${directory:log}/mariadb_stdout.log",
"stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
......@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line =
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
wrapper-path = ${directory:bin}/${:command}
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:command}
mode = 755
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--protocol=socket \
${:extra-args} "$@"
wrapper-path = ${:rendered}
extra-args =
[binary-wrap-mysql]
......@@ -208,6 +272,11 @@ command = mysqladmin
<= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql
template =
inline:#!/bin/sh
exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
--defaults-file="${directory:etc}/mysql/my.cnf" \
--skip-write-binlog "$@"
[binary-wrap-pt-digest]
<= binary-wrap-base
......@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var
lib = ${:var}/lib
mysql = ${:lib}/mysql
log = ${:var}/log
run = ${:var}/run
config-tmp = ${:tmp}/config
custom = ${directory:etc}/mysql/custom
[mysql-directory]
recipe = slapos.cookbook:mkdirectory
mysql = ${directory:etc}/mysql
ssl = ${:mysql}/ssl
data = ${directory:lib}/mysql
custom = ${:mysql}/custom.d
[dbjob-parameter]
bash-bin = {{ bash_bin }}
db-user = ${init-script-parameters:root-user}
db-password = ${init-script-parameters:password}
mysql-dir = ${directory:mysql}
dbjob-cnf = ${directory:etc}/mysql/my.cnf
log-dir = ${directory:mysql}/.system/logs
mysql-dir = ${mysql-directory:data}
dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
log-dir = ${mysql-directory:data}/.system/logs
tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket}
mysql-client = ${binary-wrap-mysql:wrapper-path}
socat-port = {{ int(port) + 9 }}
restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }}
......@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
tar -xzf config.tar.gz
cp -r data/.system ${directory:mysql}
rm -rf ${directory:etc}/mysql
cp -r etc/mysql ${directory:etc}
ln -sf ${directory:mysql}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
rm -r etc/mysql/ssl # we don't need generated ssl files
cp -r data/.system ${mysql-directory:data}
cp -r etc/mysql/* ${mysql-directory:mysql}
ln -sf ${mysql-directory:data}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
[{{ section('install-mysql-config') }}]
recipe = plone.recipe.command
......
......@@ -89,33 +89,6 @@
"type": "string",
"default": ""
},
"tags": {
"title": "Provisioning db tag list",
"description": "Provisioning db tags. Set one tag per line.",
"type": "array",
"items": {
"type": "string"
},
"default": [
"gtidstrict",
"bind",
"pkg",
"innodb",
"noquerycache",
"slow",
"pfs",
"linux",
"readonly",
"diskmonitor",
"sqlerror",
"compressbinlog",
"bm4ci",
"mroonga",
"utctime",
"readcommitted",
"nohandshake"
]
},
"http-session-lifetime": {
"title": "Web Session life time in s",
"description": "Web interface Session life time in seconds. Default 86400",
......@@ -128,17 +101,11 @@
"type": "integer",
"default": 4
},
"autorejoin": {
"title": "Automatic rejoin a failed master",
"description": "Automatic rejoin a failed master (default true)",
"require-ssl": {
"title": "Enable REQUIRE SSL for repman database users",
"description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
"type": "boolean",
"default": true
},
"autoseed": {
"title": "Automatic join a standalone node",
"description": "Automatic join a standalone node",
"type": "boolean",
"default": true
"default": false
},
"repman-cluster-dict": {
"title": "Replication Manager clusters definition",
......@@ -189,6 +156,34 @@
"type": "string",
"default": ""
},
"tags": {
"title": "Provisioning db tag list",
"description": "Provisioning db tags. Set one tag per line.",
"type": "array",
"items": {
"type": "string"
},
"default": [
"gtidstrict",
"bind",
"pkg",
"innodb",
"noquerycache",
"slow",
"pfs",
"linux",
"readonly",
"diskmonitor",
"sqlerror",
"compressbinlog",
"bm4ci",
"mroonga",
"utctime",
"readcommitted",
"nohandshake",
"ssl"
]
},
"proxysql-user": {
"title": "Proxysql username",
"description": "Proxysql external user, default is 'external'.",
......@@ -197,7 +192,7 @@
},
"proxy-tags": {
"title": "Proxy tag list",
"description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]",
"description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
"type": "array",
"items": {
"type": "string"
......@@ -206,7 +201,8 @@
"pkg",
"masterslave",
"linux",
"noreadwritesplit"
"noreadwritesplit",
"ssl"
]
},
"logical-backup-cron": {
......@@ -302,6 +298,18 @@
"type": "boolean",
"default": true
},
"autorejoin": {
"title": "Automatic rejoin a failed master db",
"description": "Automatic rejoin a failed master (default: true).",
"type": "boolean",
"default": true
},
"autoseed": {
"title": "Automatic join a standalone node",
"description": "Automatic join a standalone mariadb node to the cluster.",
"type": "boolean",
"default": true
},
"failover-mode": {
"title": "Failover mode",
"description": "Failover is manual or automatic (default \"manual\").",
......
{
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"db_user": {
"description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
"title": "Database User",
"type": "string",
"default": "user"
},
"db_password": {
"description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
"title": "Initial database password",
"type": "string",
"default": ""
},
"db_charset": {
"description": "Database Character set information. This parameter is set only if database is not created yet.",
"title": "Database Character set information",
"type": "string",
"enum": [
"latin1",
"latin2",
"latin5",
"latin7",
"ascii",
"latin5",
"utf8",
"utf8mb4",
"utf16",
"utf16le",
"utf32",
"binary",
"utf8_general_ci",
"utf8_unicode_ci",
"utf8_unicode_ci",
"utf8mb4_bin",
"utf8mb4_general_ci",
"utf8mb4_unicode_ci",
"latin1_general_ci",
"latin1_bin",
"ascii_general_ci",
"ascii_bin"
],
"default": ""
}
},
"title": "Input Parameters",
"type": "object"
}
......@@ -28,8 +28,10 @@ mode = 0644
extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_}
supervisord-lib = {{ supervisord_lib }}
causace-lib = {{ caucase_library }}
import-list =
file supervisord_lib :supervisord-lib
file caucase :causace-lib
context =
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
......@@ -48,6 +50,8 @@ context =
raw bash_bin {{ bash_location }}/bin/bash
raw jq_bin {{ jq_location }}/bin/jq
raw curl_bin {{ curl_location }}/bin/curl
raw openssl_bin {{ openssl_location }}/bin/openssl
raw caucase_bin_client {{ caucase_bin_client }}
${:extra-context}
extra-context =
......@@ -56,6 +60,7 @@ extra-context =
<= jinja2-template-base
template= {{ template_repman_cfg }}
extra-context =
key slave_instance_list slap-configuration:slave-instance-list
raw gowork_bin {{ gowork_bin }}
raw haproxy_location {{ haproxy_location }}
raw nginx_bin {{ nginx_location }}/sbin/nginx
......@@ -71,6 +76,7 @@ extra-context =
raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }}
raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
raw template_publish_slave_information {{ template_publish_slave }}
[template-mariadb-parameters]
bash = {{ bash_location }}
......@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }}
template-mysqld-need-start = {{ mysqld_start_template }}
template-manage-db = {{ template_slave_db_manage }}
link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }}
......
{% set part_list = [] -%}
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
{% for _, database in slave_dict.items() -%}
{% set slave_reference = database['slave_reference'] -%}
{% do part_list.append('publish-slave-' ~ slave_reference) -%}
[publish-slave-{{ slave_reference }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
{% if database.get('user') -%}
url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
{% else -%}
url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
{% endif -%}
{% endfor %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
......@@ -18,9 +18,10 @@ extends =
../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg
../../component/jq/buildout.cfg
../../component/openssl/buildout.cfg
../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg
../../stack/caucase/buildout.cfg
parts =
slapos-cookbook
......@@ -28,14 +29,43 @@ parts =
instance.cfg
template-mariadb.cfg
template-mysqld-wrapper
caucase-eggs
gowork
[mariadb]
location = ${mariadb-10.4:location}
[python]
part = python3
[gowork]
# replication-manager does not build on golang 1.17
golang = ${golang1.16:location}
[template-mysqld-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
mode = 644
template =
inline:{% raw %}#!/bin/sh -e
basedir='${mariadb:location}'
datadir='{{datadir}}'
marker=$datadir/.slapos_initializing
system=$datadir/.system
[ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
touch "$marker"
"$basedir/scripts/mysql_install_db" \
--defaults-file='{{defaults_file}}' \
--skip-name-resolve \
--auth-root-authentication-method=normal \
--basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
--datadir="$datadir"
rm "$marker"
}
{%- if environ is defined %}
{%- for variable in environ.splitlines() %}
{{ variable }} \
{%- endfor %}
{%- endif %}
exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
{% endraw %}
[instance.cfg]
recipe = slapos.recipe.template:jinja2
......@@ -45,6 +75,7 @@ mode = 0644
context =
key bash_location bash:location
key bin_directory buildout:bin-directory
key caucase_library caucase-jinja2-library:target
key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location
......@@ -70,6 +101,7 @@ context =
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location
key openssl_location openssl:location
key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location
......@@ -87,10 +119,12 @@ context =
key template_init_root_sql mariadb-init-root-sql:target
key template_init_root_wrapper init-root-wrapper-in:target
key template_repman_cfg instance-repman.cfg:target
key template_publish_slave template-publish-slave-information:target
key template_slave_db_manage slave-db-manage.in:target
key unixodbc_location unixodbc:location
key sysbench_location sysbench:location
key proxysql_location proxysql:location
raw caucase_bin_client ${buildout:bin-directory}/caucase
[download-file]
recipe = slapos.recipe.build:download
......@@ -126,6 +160,9 @@ link-binary =
[template-mariadb-initial-setup]
<= download-file
[template-publish-slave-information]
<= download-file
[template-my-cnf]
<= download-file
......@@ -146,3 +183,6 @@ link-binary =
[proxy-need-start-stop.sh.in]
<= download-file
[slave-db-manage.in]
<= download-file
......@@ -10,6 +10,15 @@
"request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json",
"index": 0
},
"default-slave": {
"title": "Replication Manager Database",
"description": "Replication Manager Database",
"software-type": "default",
"request": "instance-repman-slave-input-schema.json",
"response": "instance-repman-output-schema.json",
"shared": true,
"index": 1
}
}
}
......@@ -12,6 +12,11 @@ monitoring-write-heartbeat-credential="{{ parameter_dict['heartbeat-user'] }}:{{
db-servers-connect-timeout = 1
slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
autoseed = {{ setbool(parameter_dict['autoseed']) }}
{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
autorejoin-logical-backup = true
{% endif -%}
proxysql = true
proxysql-port = {{ parameter_dict['proxy-port'] }}
......@@ -40,6 +45,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
#scheduler-db-servers-receiver-use-ssl ## set to true when ssl enabled on socat
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
......@@ -49,6 +55,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
replication-use-ssl = true
test-inject-traffic = true
# failover
failover-mode = "{{ parameter_dict['failover-mode'] }}"
......
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[Default]
api-bind = "{{ parameter_dict['ipv4'] }}"
http-bind-address = "{{ parameter_dict['ipv4'] }}"
......@@ -11,17 +8,14 @@ monitoring-save-config = false
api-https-bind = true
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
include = "{{ parameter_dict['cluster-d'] }}"
autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
autoseed = {{ setbool(parameter_dict['autoseed']) }}
{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
autorejoin-logical-backup = true
{% endif -%}
db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40
monitoring-wait-retry = 1200
monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
#haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
......@@ -48,6 +42,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
# Mail configuration
# Alert email sender (default "mrm@localhost")
......
#!/bin/bash
USER={{ parameter_dict['db-user'] }}
DBUSER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/
MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
{% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%}
{% else -%}
......@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
doneJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
$MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
pauseJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
$MYSQL_CLIENT -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
partialRestore()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
$MYSQL_CLIENT -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
$MYSQL_CLIENT -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
$MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
$MYSQL_CLIENT -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
$MYSQL_CLIENT -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;"
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
$MYSQL_CLIENT -e "flush privileges;start slave;"
}
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
......@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}"
do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
$MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then
echo "No $job needed"
......@@ -77,14 +79,14 @@ do
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
$MYSQL_CLIENT -e 'start slave;'
;;
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
$MYSQL_CLIENT -e 'start slave;'
;;
reseedmariabackup)
rm -rf $BACKUPDIR
......@@ -107,7 +109,7 @@ do
;;
mariabackup)
cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
......@@ -116,10 +118,10 @@ do
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
$MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
restart)
{{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
{{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
esac
doneJob
......
......@@ -2,7 +2,7 @@
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost $@
--protocol=socket -hlocalhost "$@"
}
if [ ! -f "{{ init_password_done }}" ]; then
......@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
fi
echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "remove unwanted users...";
run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
echo "done"
fi
......@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
if [ $? -eq 0 ]; then
touch {{ upgrade_done }};
fi
else
# only create databases now
run_mysql < {{ init_database_sql }} || exit 1;
fi
......@@ -2,16 +2,18 @@
-- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0;
# {{ parameter_dict["require-ssl"] }}
{% set ssl = parameter_dict["require-ssl"] -%}
CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
......
SET @@SESSION.SQL_LOG_BIN=0;
USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
{% macro database(name, user, password, ssl, charset="") -%}
{% set charset_collate = "" -%}
{% set charset_dict = {'utf8_general_ci': 'utf8',
'utf8_unicode_ci': 'utf8',
'utf8mb4_bin': 'utf8mb4',
'utf8mb4_general_ci': 'utf8mb4',
'utf8mb4_unicode_ci': 'utf8mb4',
'latin1_general_ci': 'latin1',
'latin1_bin': 'latin1',
'ascii_general_ci': 'ascii',
'ascii_bin': 'ascii'} -%}
{% if charset %}
{% set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
{% if charset_dict.get(charset) -%}
{% set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
{% endif -%}
{% endif -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %}
{% endmacro -%}
{% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }}
{{ database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
{% endfor -%}
......@@ -4,13 +4,11 @@ curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then
sleep 5
echo "$CODE: Updating mysql configuration..."
# update mysql configuration
{{ update_config }}
......@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
sleep 5
# check again if the service is still up...
{{ mariadb_controller }} status mariadb
else
echo "CODE: $CODE. No need start!"
fi
#!/bin/bash
set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
# Only write or delete on master database else, we break replication.
if [ "$DATADIR" = "{{ partition_dir }}" ]; then
cat << EOF > {{ tmp_dir }}/.script.sql
use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` (
\`name\` varchar(80) NOT NULL,
\`state\` tinyint(1) DEFAULT NULL,
PRIMARY KEY (\`name\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
UPDATE \`slave\` set \`state\`=false;
{% for name in database_name.split(' ') -%}
{% if name -%}
REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
{% endif -%}
{% endfor -%}
EOF
# Update requested slaves database
run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql
DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$?
if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME"
exit $RET
fi
if [ -z "$DBNAME" ]; then
echo "No database for slave to remove.";
fi
for NAME in $DBNAME; do
if [ ! -z "$NAME" ]; then
echo "Deleting database $NAME..."
run_mysql -e "DROP DATABASE IF EXISTS $NAME";
run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
echo "Done."
fi
done
fi
......@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
token = resp.json()['token']
headers = {"authorization": "Bearer " + token}
resp = requests.get(
urljoin(self.url, '/api/monitor'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
for i in range(20):
resp = requests.get(
......@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
self.assertTrue(cluster['isProvision'])
self.assertTrue(cluster['isFailable'])
self.assertFalse(cluster['isDown'])
resp = requests.get(
urljoin(self.url, '/api/clusters/cluster1/status'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
self.assertEqual(resp.json(), {"alive": "running"})
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 597c29546519aabe7259e416d0b92095
md5sum = ba6c572678687081ce111d3805be3cee
......@@ -35,10 +35,10 @@ repository = ${caucase-repository:location}
[caucase-test-runner]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
export HOSTS="$(mktemp)"
trap 'rm "$HOSTS"' EXIT
printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS"
export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/userhosts.so:$LD_PRELOAD
export HOSTS_FILE="$(mktemp)"
trap 'rm "$HOSTS_FILE"' EXIT
printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS_FILE"
export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/libuserhosts.so:$LD_PRELOAD
exec python -m unittest discover -v
rendered = $${caucase:location}/host_setting.sh
......
......@@ -47,7 +47,7 @@ md5sum = 1a668d6203d42b4d46d56e24c7606cb2
[python-language-server-requirements.txt]
_update_hash_filename_ = python-language-server-requirements.txt
md5sum = 9f478fd1b03b7738f3de549cb899bf54
md5sum = e16f1414f2657fa5ffc949839207f41b
[slapos.css.in]
_update_hash_filename_ = slapos.css.in
......
appdirs==1.4.4
astroid==2.5.2
attrs==20.3.0
astroid==2.6.2
attrs==21.2.0
black==20.8b1
certifi==2020.12.5
cachetools==4.2.4
certifi==2021.5.30
chardet==4.0.0
click==7.1.2
future==0.18.2
idna==2.10
importlib-metadata==3.10.0
isort==5.8.0
iniconfig==1.1.1
isort==5.9.1
jedi==0.18.0
lazy-object-proxy==1.4.3
lazy-object-proxy==1.6.0
mccabe==0.6.1
mypy==0.812
mypy-extensions==0.4.3
mypy==0.930
packaging==21.0
parso==0.8.2
pathspec==0.8.1
pluggy==0.13.1
pydantic==1.8.1
pydantic==1.8.2
pydocstyle==6.0.0
pyflags==0.1
pyflakes==2.1.0
pygls==0.10.2
pylint==2.7.4
pygls==0.11.1
pylint==2.9.3
pyparsing==2.4.7
regex==2021.4.4
requests==2.25.1
rope==0.11.0
six==1.12.0
six==1.16.0
snowballstemmer==1.2.1
toml==0.10.2
tomli==1.2.3
typed-ast==1.4.1
typeguard==2.12.0
typing-extensions==3.7.4.3
urllib3==1.26.4
wrapt==1.11.2
yapf==0.29.0
typeguard==2.12.1
typing-extensions==4.0.1
urllib3==1.26.6
wrapt==1.12.1
yapf==0.31.0
zc.buildout.languageserver==0.6.2
zc.buildout==2.13.4
zc.buildout.languageserver==0.5.0
zipp==3.4.1
zipp==3.4.1
\ No newline at end of file
......@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = c82ea00c4514b72fb97a6fa7ac36ec52
md5sum = f553aa7d6596dcf98e7e61bfb6bd81c7
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -46,7 +46,7 @@ md5sum = 1de449e8c0c4a85c5ce2b447785b7654
[template-mariadb-initial-setup]
filename = mariadb_initial_setup.sql.in
md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
md5sum = f928b9dc99f7f970caadfe7dd6f95d34
[template-postfix]
filename = instance-postfix.cfg.in
......@@ -70,7 +70,7 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
[template]
filename = instance.cfg.in
md5sum = bbef65b4edeb342f08309604ca3717d5
md5sum = 0444d646607f8643e987bb62d1cbbc1d
[template-erp5]
filename = instance-erp5.cfg.in
......@@ -86,11 +86,11 @@ md5sum = bc821f9f9696953b10a03ad7b59a1936
[template-zope]
filename = instance-zope.cfg.in
md5sum = f3121380ab4d31ba5f4984aec74d0a2f
md5sum = 09895deebca3206425fada3caf1d92cf
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = d6166515fda7b09df754672536b131be
md5sum = cbcbe2daf3a51dfab50446a18c0be76e
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
......@@ -189,7 +189,7 @@ cert = ${haproxy-conf-ssl:certificate}
ca-cert = ${haproxy-conf-ssl:ca-cert}
crl = ${haproxy-conf-ssl:crl}
{% endif %}
stats-socket = ${directory:run}/haproxy.sock
stats-socket = ${directory:run}/ha.sock
path-routing-list = {{ dumps(slapparameter_dict['path-routing-list']) }}
family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict']) }}
pidfile = ${directory:run}/haproxy.pid
......
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure', 'with-process-privilege': True}]) -%}
{% set test_database_list = [] %}
{% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
{% do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}
......
......@@ -185,7 +185,7 @@ context = key host_dict hosts-parameter:host-dict
recipe = slapos.recipe.template:jinja2
mode = 755
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
export HOSTS=${hosts:rendered}
export HOSTS_FILE=${hosts:rendered}
export LD_PRELOAD={{ parameter_dict['userhosts'] }}:$LD_PRELOAD
exec "$@"
rendered = ${directory:bin}/userhosts
......
......@@ -110,7 +110,7 @@ link-binary = {{ dumps(zope_link_binary) }}
fonts = {{ dumps(zope_fonts) }}
fontconfig-includes = {{ dumps(zope_fontconfig_includes) }}
template-fonts-conf = {{ dumps(template_fonts_conf) }}
userhosts = {{ userhosts_location }}/lib/userhosts.so
userhosts = {{ userhosts_location }}/lib/libuserhosts.so
site-zcml = {{ site_zcml }}
extra-path-list = {{ dumps(extra_path_list) }}
matplotlibrc = {{ matplotlibrc_location }}
......
......@@ -4,16 +4,19 @@ USE mysql;
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%}
{% macro database(name, user, password, with_process_privilege) -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
{% if with_process_privilege %}
GRANT PROCESS ON *.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT PROCESS ON *.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
{%- endif %}
{%- endif %}
{% endmacro -%}
{% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }}
{{ database(entry['name'], entry.get('user'), entry.get('password'), entry.get('with-process-privilege')) }}
{% endfor -%}
......@@ -132,7 +132,7 @@ eggs =
[versions]
setuptools = 44.1.1
# Use SlapOS patched zc.buildout
zc.buildout = 2.7.1+slapos014
zc.buildout = 2.7.1+slapos015
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
......@@ -167,7 +167,7 @@ importlib-metadata = 1.7.0:whl
inotify-simple = 1.1.1
itsdangerous = 0.24
lock-file = 2.0
lxml = 4.6.3
lxml = 4.6.5
meld3 = 1.0.2
mock = 3.0.5
more-itertools = 5.0.0
......@@ -189,14 +189,14 @@ setproctitle = 1.1.10
setuptools-dso = 1.7
rubygemsrecipe = 0.4.3
six = 1.12.0
slapos.cookbook = 1.0.220
slapos.cookbook = 1.0.226
slapos.core = 1.7.2
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.22
slapos.rebootstrap = 4.5
slapos.recipe.build = 0.52
slapos.recipe.cmmi = 0.18
slapos.recipe.build = 0.53
slapos.recipe.cmmi = 0.19
slapos.recipe.template = 4.6
slapos.toolbox = 0.126
stevedore = 1.21.0:whl
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment